@datafuel/sdk 0.2.0 → 0.4.0

package/dist/index.d.cts

@@ -40,7 +40,9 @@ interface AI {
     prompt?: string;
     /** Example JSON object the output must follow. */
     format?: unknown;
-    provider?: "openai" | "anthropic" | "google";
+    /** Required. One of the providers `df.aiProviders()` lists; the API answers 400 INVALID_ATTRIBUTES without it. */
+    provider?: string;
+    /** One of the models `df.aiProviders()` lists for `provider`. Omit for the provider's default. */
     model?: string;
     apiKey?: string;
 }
@@ -54,9 +56,15 @@ interface ScrapeOptions {
     jsRendering?: boolean;
     waitFor?: string;
     waitForTimeoutMs?: number;
-    /** An object keyed by action, e.g. `{ click: "#more" }`. `df.jsInstructions()` lists them. */
+    /**
+     * Browser actions run after load: an array of single-action objects, run in
+     * order, e.g. `[{ click: "#more" }, { wait_ms: 1000 }, { click: "#more" }]`.
+     * An object keyed by action is still accepted, but its order is not
+     * guaranteed and an action cannot repeat. `df.jsInstructions()` lists them.
+     */
     jsInstructions?: unknown;
-    blockResource?: string;
+    /** Resource types the browser must not load: one, e.g. `"Image"`, or several. */
+    blockResource?: string | string[];
     /** Markdown only: always render just the `<main>` / `<article>` container. */
     mainContentOnly?: boolean;
     /** Markdown only: `false` drops images and saves tokens. */
@@ -310,6 +318,8 @@ interface TransactionsOptions {
 interface Transaction {
     id: number;
     amount: number;
+    /** The part of `amount` that moved plan credits, same sign; the rest moved pay-as-you-go credits. */
+    plan_amount?: number;
     operation: TransactionOperation;
     /** What `reference_id` points to, e.g. `task_id` or `job_id`. */
     reference_type: string;
@@ -400,13 +410,29 @@ interface Analytics {
     })[];
     by_status_code: StatusCodeBreakdown[];
 }
+/**
+ * Credits by pool. Plan credits are spent first, roll over when the plan renews and
+ * expire if it is not renewed. Pay-as-you-go credits come from credit packs, are spent
+ * after plan credits and never expire.
+ */
+interface BalanceSplit {
+    /** Total spendable credits: `plan_balance` plus `payg_balance`. */
+    balance: number;
+    plan_balance: number;
+    payg_balance: number;
+}
 /** The account behind the API key. */
 interface Profile {
     email: string;
     username: string;
     current_concurrency: number;
     concurrency_limit: number;
+    /** Total spendable credits: `plan_credit_balance` plus `payg_credit_balance`. */
     credit_balance: number;
+    /** Spent first; expire if the plan is not renewed. */
+    plan_credit_balance?: number;
+    /** Spent after plan credits; never expire. */
+    payg_credit_balance?: number;
     monthly_credit_limit: number;
 }
 /** One argument of a browser action. */
@@ -427,6 +453,28 @@ interface JsInstruction {
     iframe: boolean;
     example: unknown;
 }
+/** An LLM provider `ai.provider` accepts, with the models `ai.model` accepts for it. */
+interface AIProvider {
+    name: string;
+    models: string[];
+}
+/** The anti-bot protection in front of one host and path, from `checkProtection`. */
+interface ProtectionCheck {
+    host: string;
+    path: string;
+    /** cloudflare, cloudflare_5sec, akamai, imperva, perimeterx or unprotected. */
+    protection_type: string;
+}
+/** The state of the API. `checks` is filled by a deep check only, keyed by dependency. */
+interface Health {
+    status: "ok" | "degraded" | (string & {});
+    checks?: Record<string, {
+        status: "ok" | "fail" | (string & {});
+        latency_ms: number;
+    }>;
+    /** Whether the API can serve requests. */
+    ok: boolean;
+}
 /** A named proxy location: a city, or an ASN. */
 interface ProxyLocation {
     code: string;
@@ -470,7 +518,7 @@ declare class Capabilities {
 
 declare const DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
 /** Kept in step with package.json by a test; see test/hardening.test.ts. */
-declare const VERSION = "0.2.0";
+declare const VERSION = "0.4.0";
 /** Options for the llm_scraping module, which reads its proxy country here. */
 interface AskOptions {
     engine: Engine | string;
@@ -538,7 +586,10 @@ interface SearchOptions {
     ludocid?: string;
     lsig?: string;
     ibp?: string;
-    /** Exit country of the request. */
+    /**
+     * Accepted but not used yet: searches leave through DataFuel's own pool.
+     * Target a market with `country` and `language`.
+     */
     proxyCountry?: string;
     /** Default json. */
     format?: "json" | "html" | "markdown";
@@ -721,6 +772,21 @@ declare class DataFuel {
     capabilities(options?: CallOptions): Promise<Capabilities>;
     /** The browser actions `jsInstructions` accepts, with their arguments. Needs no key. */
     jsInstructions(options?: CallOptions): Promise<JsInstruction[]>;
+    /** The LLM providers and models `ai` accepts. Needs no key. */
+    aiProviders(options?: CallOptions): Promise<AIProvider[]>;
+    /**
+     * Whether the API is up. `deep` also checks the dependencies it needs to
+     * serve scrapes. A degraded report is returned, not thrown: read `ok`.
+     * Needs no key.
+     */
+    health(options?: CallOptions & {
+        deep?: boolean;
+    }): Promise<Health>;
+    /**
+     * Which anti-bot protection sits in front of each URL. Nothing is scraped
+     * and nothing is charged; invalid URLs are skipped.
+     */
+    checkProtection(urls: string[], options?: CallOptions): Promise<ProtectionCheck[]>;
     /** Countries, regions and cities a proxy type can exit from. */
     proxyLocations(options?: CallOptions & {
         proxyType?: string;
@@ -729,10 +795,12 @@ declare class DataFuel {
     proxyAsns(country: string, options?: CallOptions & {
         proxyType?: string;
     }): Promise<ProxyLocation[]>;
-    /** Remaining credits. */
+    /** Remaining credits: plan and pay-as-you-go together. */
     balance(options?: CallOptions): Promise<number>;
+    /** Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits. */
+    balanceSplit(options?: CallOptions): Promise<BalanceSplit>;
     /**
-     * Credit movements, newest first: purchases, usage, refunds, expiry.
+     * Credit movements, newest first: plan assignments, credit pack purchases, usage, refunds, expiry.
      * `sums` totals each operation over the whole range, not just this page.
      */
     transactions(options?: TransactionsOptions & CallOptions): Promise<TransactionsPage>;
@@ -772,7 +840,7 @@ declare class NoApiKey extends DataFuelError {
 declare class TransportError extends DataFuelError {
 }
 /** The `code` of an API error. The API may add codes; unknown ones pass through. */
-type ErrorCode = "UNAUTHORIZED" | "INVALID_API_KEY" | "FORBIDDEN" | "INSUFFICIENT_CREDITS" | "RATE_LIMIT_EXCEEDED" | "CONCURRENCY_LIMIT_REACHED" | "INVALID_REQUEST_BODY" | "INVALID_ATTRIBUTES" | "MISSING_TARGET" | "UNSUPPORTED_TASK_TYPE" | "JOB_REQUIRES_MULTIPLE_TARGETS" | "INVALID_IDEMPOTENCY_KEY" | "IDEMPOTENCY_KEY_REUSED" | "INVALID_TASK_ID" | "INVALID_JOB_ID" | "TASK_NOT_FOUND" | "JOB_NOT_FOUND" | "CRAWL_NOT_FOUND" | "JOB_NOT_CANCELLABLE" | "INVALID_CRAWL_PATTERN" | "CRAWL_UNSUPPORTED_OPTION" | "INVALID_CURSOR" | "INVALID_QUERY_PARAM" | "INVALID_PROXY_TYPE" | "INVALID_COUNTRY" | "INVALID_DATE_FORMAT" | "INVALID_DATE_RANGE" | "INVALID_INTERVAL" | "MODULE_UNAVAILABLE" | "ENGINE_UNAVAILABLE" | "API_KEY_RESET_FAILED" | "TASK_RESULT_TIMEOUT" | "INTERNAL_ERROR" | (string & {});
+type ErrorCode = "UNAUTHORIZED" | "INVALID_API_KEY" | "FORBIDDEN" | "INSUFFICIENT_CREDITS" | "RATE_LIMIT_EXCEEDED" | "CONCURRENCY_LIMIT_REACHED" | "INVALID_REQUEST_BODY" | "INVALID_ATTRIBUTES" | "MISSING_TARGET" | "UNSUPPORTED_TASK_TYPE" | "JOB_REQUIRES_MULTIPLE_TARGETS" | "INVALID_IDEMPOTENCY_KEY" | "IDEMPOTENCY_KEY_REUSED" | "INVALID_TASK_ID" | "INVALID_JOB_ID" | "TASK_NOT_FOUND" | "JOB_NOT_FOUND" | "CRAWL_NOT_FOUND" | "JOB_NOT_CANCELLABLE" | "TASK_ALREADY_EXISTS" | "JOB_ALREADY_EXISTS" | "URL_LENGTH_CANNOT_BE_ZERO" | "ANALYTICS_FETCH_FAILED" | "INVALID_CRAWL_PATTERN" | "CRAWL_UNSUPPORTED_OPTION" | "INVALID_CURSOR" | "INVALID_QUERY_PARAM" | "INVALID_PROXY_TYPE" | "INVALID_COUNTRY" | "INVALID_DATE_FORMAT" | "INVALID_DATE_RANGE" | "INVALID_INTERVAL" | "MODULE_UNAVAILABLE" | "ENGINE_UNAVAILABLE" | "API_KEY_RESET_FAILED" | "TASK_RESULT_TIMEOUT" | "INTERNAL_ERROR" | (string & {});
 /** A non-2xx answer from the API itself. */
 declare class APIError extends DataFuelError {
     readonly status: number;
@@ -805,6 +873,13 @@ declare class IdempotencyKeyReused extends APIError {
 /** 409 JOB_NOT_CANCELLABLE: the job or crawl already finished. */
 declare class JobNotCancellable extends APIError {
 }
+/**
+ * 409 TASK_ALREADY_EXISTS / JOB_ALREADY_EXISTS: the create collided with an
+ * existing task or job and is not an idempotent replay. Nothing was charged;
+ * send the request again.
+ */
+declare class AlreadyExists extends APIError {
+}
 /** 503: an operator switched something off, or a dependency is down. The message carries the reason. */
 declare class Unavailable extends APIError {
 }
@@ -844,4 +919,4 @@ declare class WaitTimeout extends DataFuelError {
     constructor(message: string, id?: string, status?: unknown);
 }
 
-export { type AI, APIError, type Analytics, type AnalyticsCounts, type AnalyticsOptions, type AskOptions, Blocked, type CallOptions, type CancelResult, Capabilities, type Capability, type ClientOptions, type CrawlOptions, CrawlPage, type CrawlResult, type CrawlResultsPage, type CrawlStatus, DEFAULT_BASE_URL, DataFuel, DataFuelError, type Engine, EngineUnavailable, type ErrorCode, Forbidden, type Format, IdempotencyKeyReused, InsufficientCredits, InvalidAttributes, JobNotCancellable, type JobResults, type JobStatus, type JobSummary, type JobsPage, type JsInstruction, type JsInstructionArg, type Link, type ListOptions, type ListTasksOptions, type MapOptions, ModuleUnavailable, NoApiKey, NotFound, type Payload, type PreviousPeriod, type Profile, type Proxy, type ProxyCountry, type ProxyLocation, type ProxyType, RateLimited, Result, type ScrapeOptions, type SearchOptions, type SiteMap, type Status, type StatusCodeBreakdown, TaskFailed, type TaskSummary, type TaskType, type TasksPage, type Transaction, type TransactionOperation, type TransactionSum, type TransactionsOptions, type TransactionsPage, TransportError, Unauthorized, Unavailable, VERSION, WaitTimeout, isDone };
+export { type AI, type AIProvider, APIError, AlreadyExists, type Analytics, type AnalyticsCounts, type AnalyticsOptions, type AskOptions, type BalanceSplit, Blocked, type CallOptions, type CancelResult, Capabilities, type Capability, type ClientOptions, type CrawlOptions, CrawlPage, type CrawlResult, type CrawlResultsPage, type CrawlStatus, DEFAULT_BASE_URL, DataFuel, DataFuelError, type Engine, EngineUnavailable, type ErrorCode, Forbidden, type Format, type Health, IdempotencyKeyReused, InsufficientCredits, InvalidAttributes, JobNotCancellable, type JobResults, type JobStatus, type JobSummary, type JobsPage, type JsInstruction, type JsInstructionArg, type Link, type ListOptions, type ListTasksOptions, type MapOptions, ModuleUnavailable, NoApiKey, NotFound, type Payload, type PreviousPeriod, type Profile, type ProtectionCheck, type Proxy, type ProxyCountry, type ProxyLocation, type ProxyType, RateLimited, Result, type ScrapeOptions, type SearchOptions, type SiteMap, type Status, type StatusCodeBreakdown, TaskFailed, type TaskSummary, type TaskType, type TasksPage, type Transaction, type TransactionOperation, type TransactionSum, type TransactionsOptions, type TransactionsPage, TransportError, Unauthorized, Unavailable, VERSION, WaitTimeout, isDone };

package/dist/index.d.ts

@@ -40,7 +40,9 @@ interface AI {
     prompt?: string;
     /** Example JSON object the output must follow. */
     format?: unknown;
-    provider?: "openai" | "anthropic" | "google";
+    /** Required. One of the providers `df.aiProviders()` lists; the API answers 400 INVALID_ATTRIBUTES without it. */
+    provider?: string;
+    /** One of the models `df.aiProviders()` lists for `provider`. Omit for the provider's default. */
     model?: string;
     apiKey?: string;
 }
@@ -54,9 +56,15 @@ interface ScrapeOptions {
     jsRendering?: boolean;
     waitFor?: string;
     waitForTimeoutMs?: number;
-    /** An object keyed by action, e.g. `{ click: "#more" }`. `df.jsInstructions()` lists them. */
+    /**
+     * Browser actions run after load: an array of single-action objects, run in
+     * order, e.g. `[{ click: "#more" }, { wait_ms: 1000 }, { click: "#more" }]`.
+     * An object keyed by action is still accepted, but its order is not
+     * guaranteed and an action cannot repeat. `df.jsInstructions()` lists them.
+     */
     jsInstructions?: unknown;
-    blockResource?: string;
+    /** Resource types the browser must not load: one, e.g. `"Image"`, or several. */
+    blockResource?: string | string[];
     /** Markdown only: always render just the `<main>` / `<article>` container. */
     mainContentOnly?: boolean;
     /** Markdown only: `false` drops images and saves tokens. */
@@ -310,6 +318,8 @@ interface TransactionsOptions {
 interface Transaction {
     id: number;
     amount: number;
+    /** The part of `amount` that moved plan credits, same sign; the rest moved pay-as-you-go credits. */
+    plan_amount?: number;
     operation: TransactionOperation;
     /** What `reference_id` points to, e.g. `task_id` or `job_id`. */
     reference_type: string;
@@ -400,13 +410,29 @@ interface Analytics {
     })[];
     by_status_code: StatusCodeBreakdown[];
 }
+/**
+ * Credits by pool. Plan credits are spent first, roll over when the plan renews and
+ * expire if it is not renewed. Pay-as-you-go credits come from credit packs, are spent
+ * after plan credits and never expire.
+ */
+interface BalanceSplit {
+    /** Total spendable credits: `plan_balance` plus `payg_balance`. */
+    balance: number;
+    plan_balance: number;
+    payg_balance: number;
+}
 /** The account behind the API key. */
 interface Profile {
     email: string;
     username: string;
     current_concurrency: number;
     concurrency_limit: number;
+    /** Total spendable credits: `plan_credit_balance` plus `payg_credit_balance`. */
     credit_balance: number;
+    /** Spent first; expire if the plan is not renewed. */
+    plan_credit_balance?: number;
+    /** Spent after plan credits; never expire. */
+    payg_credit_balance?: number;
     monthly_credit_limit: number;
 }
 /** One argument of a browser action. */
@@ -427,6 +453,28 @@ interface JsInstruction {
     iframe: boolean;
     example: unknown;
 }
+/** An LLM provider `ai.provider` accepts, with the models `ai.model` accepts for it. */
+interface AIProvider {
+    name: string;
+    models: string[];
+}
+/** The anti-bot protection in front of one host and path, from `checkProtection`. */
+interface ProtectionCheck {
+    host: string;
+    path: string;
+    /** cloudflare, cloudflare_5sec, akamai, imperva, perimeterx or unprotected. */
+    protection_type: string;
+}
+/** The state of the API. `checks` is filled by a deep check only, keyed by dependency. */
+interface Health {
+    status: "ok" | "degraded" | (string & {});
+    checks?: Record<string, {
+        status: "ok" | "fail" | (string & {});
+        latency_ms: number;
+    }>;
+    /** Whether the API can serve requests. */
+    ok: boolean;
+}
 /** A named proxy location: a city, or an ASN. */
 interface ProxyLocation {
     code: string;
@@ -470,7 +518,7 @@ declare class Capabilities {
 
 declare const DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
 /** Kept in step with package.json by a test; see test/hardening.test.ts. */
-declare const VERSION = "0.2.0";
+declare const VERSION = "0.4.0";
 /** Options for the llm_scraping module, which reads its proxy country here. */
 interface AskOptions {
     engine: Engine | string;
@@ -538,7 +586,10 @@ interface SearchOptions {
     ludocid?: string;
     lsig?: string;
     ibp?: string;
-    /** Exit country of the request. */
+    /**
+     * Accepted but not used yet: searches leave through DataFuel's own pool.
+     * Target a market with `country` and `language`.
+     */
     proxyCountry?: string;
     /** Default json. */
     format?: "json" | "html" | "markdown";
@@ -721,6 +772,21 @@ declare class DataFuel {
     capabilities(options?: CallOptions): Promise<Capabilities>;
     /** The browser actions `jsInstructions` accepts, with their arguments. Needs no key. */
     jsInstructions(options?: CallOptions): Promise<JsInstruction[]>;
+    /** The LLM providers and models `ai` accepts. Needs no key. */
+    aiProviders(options?: CallOptions): Promise<AIProvider[]>;
+    /**
+     * Whether the API is up. `deep` also checks the dependencies it needs to
+     * serve scrapes. A degraded report is returned, not thrown: read `ok`.
+     * Needs no key.
+     */
+    health(options?: CallOptions & {
+        deep?: boolean;
+    }): Promise<Health>;
+    /**
+     * Which anti-bot protection sits in front of each URL. Nothing is scraped
+     * and nothing is charged; invalid URLs are skipped.
+     */
+    checkProtection(urls: string[], options?: CallOptions): Promise<ProtectionCheck[]>;
     /** Countries, regions and cities a proxy type can exit from. */
     proxyLocations(options?: CallOptions & {
         proxyType?: string;
@@ -729,10 +795,12 @@ declare class DataFuel {
     proxyAsns(country: string, options?: CallOptions & {
         proxyType?: string;
     }): Promise<ProxyLocation[]>;
-    /** Remaining credits. */
+    /** Remaining credits: plan and pay-as-you-go together. */
     balance(options?: CallOptions): Promise<number>;
+    /** Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits. */
+    balanceSplit(options?: CallOptions): Promise<BalanceSplit>;
     /**
-     * Credit movements, newest first: purchases, usage, refunds, expiry.
+     * Credit movements, newest first: plan assignments, credit pack purchases, usage, refunds, expiry.
      * `sums` totals each operation over the whole range, not just this page.
      */
     transactions(options?: TransactionsOptions & CallOptions): Promise<TransactionsPage>;
@@ -772,7 +840,7 @@ declare class NoApiKey extends DataFuelError {
 declare class TransportError extends DataFuelError {
 }
 /** The `code` of an API error. The API may add codes; unknown ones pass through. */
-type ErrorCode = "UNAUTHORIZED" | "INVALID_API_KEY" | "FORBIDDEN" | "INSUFFICIENT_CREDITS" | "RATE_LIMIT_EXCEEDED" | "CONCURRENCY_LIMIT_REACHED" | "INVALID_REQUEST_BODY" | "INVALID_ATTRIBUTES" | "MISSING_TARGET" | "UNSUPPORTED_TASK_TYPE" | "JOB_REQUIRES_MULTIPLE_TARGETS" | "INVALID_IDEMPOTENCY_KEY" | "IDEMPOTENCY_KEY_REUSED" | "INVALID_TASK_ID" | "INVALID_JOB_ID" | "TASK_NOT_FOUND" | "JOB_NOT_FOUND" | "CRAWL_NOT_FOUND" | "JOB_NOT_CANCELLABLE" | "INVALID_CRAWL_PATTERN" | "CRAWL_UNSUPPORTED_OPTION" | "INVALID_CURSOR" | "INVALID_QUERY_PARAM" | "INVALID_PROXY_TYPE" | "INVALID_COUNTRY" | "INVALID_DATE_FORMAT" | "INVALID_DATE_RANGE" | "INVALID_INTERVAL" | "MODULE_UNAVAILABLE" | "ENGINE_UNAVAILABLE" | "API_KEY_RESET_FAILED" | "TASK_RESULT_TIMEOUT" | "INTERNAL_ERROR" | (string & {});
+type ErrorCode = "UNAUTHORIZED" | "INVALID_API_KEY" | "FORBIDDEN" | "INSUFFICIENT_CREDITS" | "RATE_LIMIT_EXCEEDED" | "CONCURRENCY_LIMIT_REACHED" | "INVALID_REQUEST_BODY" | "INVALID_ATTRIBUTES" | "MISSING_TARGET" | "UNSUPPORTED_TASK_TYPE" | "JOB_REQUIRES_MULTIPLE_TARGETS" | "INVALID_IDEMPOTENCY_KEY" | "IDEMPOTENCY_KEY_REUSED" | "INVALID_TASK_ID" | "INVALID_JOB_ID" | "TASK_NOT_FOUND" | "JOB_NOT_FOUND" | "CRAWL_NOT_FOUND" | "JOB_NOT_CANCELLABLE" | "TASK_ALREADY_EXISTS" | "JOB_ALREADY_EXISTS" | "URL_LENGTH_CANNOT_BE_ZERO" | "ANALYTICS_FETCH_FAILED" | "INVALID_CRAWL_PATTERN" | "CRAWL_UNSUPPORTED_OPTION" | "INVALID_CURSOR" | "INVALID_QUERY_PARAM" | "INVALID_PROXY_TYPE" | "INVALID_COUNTRY" | "INVALID_DATE_FORMAT" | "INVALID_DATE_RANGE" | "INVALID_INTERVAL" | "MODULE_UNAVAILABLE" | "ENGINE_UNAVAILABLE" | "API_KEY_RESET_FAILED" | "TASK_RESULT_TIMEOUT" | "INTERNAL_ERROR" | (string & {});
 /** A non-2xx answer from the API itself. */
 declare class APIError extends DataFuelError {
     readonly status: number;
@@ -805,6 +873,13 @@ declare class IdempotencyKeyReused extends APIError {
 /** 409 JOB_NOT_CANCELLABLE: the job or crawl already finished. */
 declare class JobNotCancellable extends APIError {
 }
+/**
+ * 409 TASK_ALREADY_EXISTS / JOB_ALREADY_EXISTS: the create collided with an
+ * existing task or job and is not an idempotent replay. Nothing was charged;
+ * send the request again.
+ */
+declare class AlreadyExists extends APIError {
+}
 /** 503: an operator switched something off, or a dependency is down. The message carries the reason. */
 declare class Unavailable extends APIError {
 }
@@ -844,4 +919,4 @@ declare class WaitTimeout extends DataFuelError {
     constructor(message: string, id?: string, status?: unknown);
 }
 
-export { type AI, APIError, type Analytics, type AnalyticsCounts, type AnalyticsOptions, type AskOptions, Blocked, type CallOptions, type CancelResult, Capabilities, type Capability, type ClientOptions, type CrawlOptions, CrawlPage, type CrawlResult, type CrawlResultsPage, type CrawlStatus, DEFAULT_BASE_URL, DataFuel, DataFuelError, type Engine, EngineUnavailable, type ErrorCode, Forbidden, type Format, IdempotencyKeyReused, InsufficientCredits, InvalidAttributes, JobNotCancellable, type JobResults, type JobStatus, type JobSummary, type JobsPage, type JsInstruction, type JsInstructionArg, type Link, type ListOptions, type ListTasksOptions, type MapOptions, ModuleUnavailable, NoApiKey, NotFound, type Payload, type PreviousPeriod, type Profile, type Proxy, type ProxyCountry, type ProxyLocation, type ProxyType, RateLimited, Result, type ScrapeOptions, type SearchOptions, type SiteMap, type Status, type StatusCodeBreakdown, TaskFailed, type TaskSummary, type TaskType, type TasksPage, type Transaction, type TransactionOperation, type TransactionSum, type TransactionsOptions, type TransactionsPage, TransportError, Unauthorized, Unavailable, VERSION, WaitTimeout, isDone };
+export { type AI, type AIProvider, APIError, AlreadyExists, type Analytics, type AnalyticsCounts, type AnalyticsOptions, type AskOptions, type BalanceSplit, Blocked, type CallOptions, type CancelResult, Capabilities, type Capability, type ClientOptions, type CrawlOptions, CrawlPage, type CrawlResult, type CrawlResultsPage, type CrawlStatus, DEFAULT_BASE_URL, DataFuel, DataFuelError, type Engine, EngineUnavailable, type ErrorCode, Forbidden, type Format, type Health, IdempotencyKeyReused, InsufficientCredits, InvalidAttributes, JobNotCancellable, type JobResults, type JobStatus, type JobSummary, type JobsPage, type JsInstruction, type JsInstructionArg, type Link, type ListOptions, type ListTasksOptions, type MapOptions, ModuleUnavailable, NoApiKey, NotFound, type Payload, type PreviousPeriod, type Profile, type ProtectionCheck, type Proxy, type ProxyCountry, type ProxyLocation, type ProxyType, RateLimited, Result, type ScrapeOptions, type SearchOptions, type SiteMap, type Status, type StatusCodeBreakdown, TaskFailed, type TaskSummary, type TaskType, type TasksPage, type Transaction, type TransactionOperation, type TransactionSum, type TransactionsOptions, type TransactionsPage, TransportError, Unauthorized, Unavailable, VERSION, WaitTimeout, isDone };

package/dist/index.js

@@ -37,6 +37,8 @@ var IdempotencyKeyReused = class extends APIError {
 };
 var JobNotCancellable = class extends APIError {
 };
+var AlreadyExists = class extends APIError {
+};
 var Unavailable = class extends APIError {
 };
 var ModuleUnavailable = class extends Unavailable {
@@ -74,14 +76,15 @@ var BY_CODE = {
   IDEMPOTENCY_KEY_REUSED: IdempotencyKeyReused,
   INVALID_API_KEY: Unauthorized,
   FORBIDDEN: Forbidden,
-  JOB_NOT_CANCELLABLE: JobNotCancellable
+  JOB_NOT_CANCELLABLE: JobNotCancellable,
+  TASK_ALREADY_EXISTS: AlreadyExists,
+  JOB_ALREADY_EXISTS: AlreadyExists
 };
 var BY_STATUS = {
   401: Unauthorized,
   402: InsufficientCredits,
   403: Forbidden,
   404: NotFound,
-  409: JobNotCancellable,
   422: IdempotencyKeyReused,
   429: RateLimited,
   503: Unavailable
@@ -102,19 +105,20 @@ function apiError(status, body, retryAfter = 0) {
 
 // src/core.ts
 var DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
-var VERSION = "0.2.0";
+var VERSION = "0.4.0";
 var DEFAULT_TIMEOUT_MS = 18e4;
 var STILL_PROCESSING_DELAY_MS = 2e3;
 var STILL_PROCESSING = "TASK_STILL_PROCESSING";
 var MAX_IDEMPOTENCY_KEY = 255;
 var Request = class {
-  constructor(method, path, params, body, idempotencyKey, auth = true) {
+  constructor(method, path, params, body, idempotencyKey, auth = true, degradedOk = false) {
     this.method = method;
     this.path = path;
     this.params = params;
     this.body = body;
     this.idempotencyKey = idempotencyKey;
     this.auth = auth;
+    this.degradedOk = degradedOk;
   }
   method;
   path;
@@ -122,6 +126,7 @@ var Request = class {
   body;
   idempotencyKey;
   auth;
+  degradedOk;
   /** GETs are safe by nature, writes because they carry an idempotency key. */
   get retryable() {
     return this.method === "GET" || this.idempotencyKey !== void 0;
@@ -169,15 +174,8 @@ function aiAttributes(ai) {
   return out;
 }
 function validateAI(ai) {
-  const given = {
-    provider: ai.provider !== void 0,
-    model: ai.model !== void 0,
-    apiKey: ai.apiKey !== void 0
-  };
-  const present = Object.values(given).filter(Boolean).length;
-  if (present > 0 && present < 3) {
-    const missing = Object.entries(given).filter(([, ok]) => !ok).map(([name]) => name).join(", ");
-    throw new TypeError(`ai needs provider, model and apiKey together; missing: ${missing}`);
+  if (!ai.provider) {
+    throw new TypeError("ai needs a provider; aiProviders() lists the ones the API supports");
   }
 }
 function scrapeAttributes(options = {}) {
@@ -187,7 +185,9 @@ function scrapeAttributes(options = {}) {
   if (options.waitFor) attrs.wait_for_selector = options.waitFor;
   if (options.waitForTimeoutMs) attrs.wait_for_selector_timeout_ms = options.waitForTimeoutMs;
   if (options.jsInstructions) attrs.js_instructions = options.jsInstructions;
-  if (options.blockResource) attrs.block_resource = options.blockResource;
+  if (options.blockResource?.length) {
+    attrs.block_resource = typeof options.blockResource === "string" ? options.blockResource : [...options.blockResource];
+  }
   if (options.mainContentOnly) attrs.main_content_only = true;
   if (options.includeImages !== void 0) attrs.include_images = options.includeImages;
   if (options.extract) attrs.extract_selector = selector(options.extract);
@@ -369,6 +369,17 @@ function crawlResultsRequest(crawlId, cursor, limit) {
     Object.keys(params).length > 0 ? params : void 0
   );
 }
+function healthRequest(deep) {
+  return new Request(
+    "GET",
+    "/healthz",
+    deep ? { deep: "1" } : void 0,
+    void 0,
+    void 0,
+    false,
+    true
+  );
+}
 function day(value) {
   return typeof value === "string" ? value : value.toISOString().slice(0, 10);
 }
@@ -601,7 +612,7 @@ var DataFuel = class {
         const signal = this.signalFor(opts);
         const send = this.fetchImpl;
         const response = await send(url, signal ? { ...init, signal } : init);
-        return await parse(response);
+        return await parse(response, request.degradedOk);
       } catch (caught) {
         if (isAbort(caught)) {
           throw new TransportError("the request was aborted or timed out", { cause: caught });
@@ -963,6 +974,37 @@ var DataFuel = class {
     const body = record(await this.send(request, options));
     return Array.isArray(body.instructions) ? body.instructions : [];
   }
+  /** The LLM providers and models `ai` accepts. Needs no key. */
+  async aiProviders(options = {}) {
+    const request = new Request(
+      "GET",
+      "/config/ai-providers",
+      void 0,
+      void 0,
+      void 0,
+      false
+    );
+    const body = record(await this.send(request, options));
+    return Array.isArray(body.providers) ? body.providers : [];
+  }
+  /**
+   * Whether the API is up. `deep` also checks the dependencies it needs to
+   * serve scrapes. A degraded report is returned, not thrown: read `ok`.
+   * Needs no key.
+   */
+  async health(options = {}) {
+    const body = record(await this.send(healthRequest(options.deep ?? false), options));
+    return { ...body, ok: body.status === "ok" };
+  }
+  /**
+   * Which anti-bot protection sits in front of each URL. Nothing is scraped
+   * and nothing is charged; invalid URLs are skipped.
+   */
+  async checkProtection(urls, options = {}) {
+    return list(
+      await this.send(new Request("POST", "/filter/check", void 0, [...urls]), options)
+    );
+  }
   /** Countries, regions and cities a proxy type can exit from. */
   async proxyLocations(options = {}) {
     const params = options.proxyType ? { proxy_type: options.proxyType } : void 0;
@@ -978,13 +1020,22 @@ var DataFuel = class {
       await this.send(new Request("GET", "/config/proxy/asn", params), options)
     );
   }
-  /** Remaining credits. */
+  /** Remaining credits: plan and pay-as-you-go together. */
   async balance(options = {}) {
     const body = await this.send(new Request("GET", "/users/@me/balance"), options);
     return intField(body, "balance");
   }
+  /** Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits. */
+  async balanceSplit(options = {}) {
+    const body = await this.send(new Request("GET", "/users/@me/balance"), options);
+    return {
+      balance: intField(body, "balance"),
+      plan_balance: intField(body, "plan_balance"),
+      payg_balance: intField(body, "payg_balance")
+    };
+  }
   /**
-   * Credit movements, newest first: purchases, usage, refunds, expiry.
+   * Credit movements, newest first: plan assignments, credit pack purchases, usage, refunds, expiry.
    * `sums` totals each operation over the whole range, not just this page.
    */
   async transactions(options = {}) {
@@ -1025,7 +1076,7 @@ var DataFuel = class {
 function envApiKey() {
   return typeof process !== "undefined" ? process.env?.DATAFUEL_API_KEY : void 0;
 }
-async function parse(response) {
+async function parse(response, degradedOk = false) {
   const text = await response.text();
   let body;
   if (text.length > 0) {
@@ -1035,6 +1086,7 @@ async function parse(response) {
       body = text;
     }
   }
+  if (degradedOk && response.status === 503 && isHealthReport(body)) return body;
   if (!response.ok) {
     const header = response.headers.get("Retry-After");
     const retryAfter = header !== null && !Number.isNaN(Number(header)) ? Number(header) : 0;
@@ -1042,6 +1094,9 @@ async function parse(response) {
   }
   return body;
 }
+function isHealthReport(body) {
+  return body !== null && typeof body === "object" && "status" in body;
+}
 function isAbort(error) {
   return error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError");
 }
@@ -1098,6 +1153,7 @@ function intField(body, name) {
 }
 export {
   APIError,
+  AlreadyExists,
   Blocked,
   Capabilities,
   CrawlPage,