@datafuel/sdk 0.1.0 → 0.3.0

package/dist/index.d.ts

@@ -24,7 +24,8 @@ interface Proxy {
     asn?: string;
     /**
      * Sticky session: the same exit across requests, `ttl` in seconds. Read by
-     * `scrape` and `map` only, and sent in attributes rather than the envelope.
+     * `scrape`, `map`, URL jobs and crawls, and sent in attributes rather than
+     * the envelope.
      */
     sessionId?: string;
     ttl?: number;
@@ -53,6 +54,7 @@ interface ScrapeOptions {
     jsRendering?: boolean;
     waitFor?: string;
     waitForTimeoutMs?: number;
+    /** An object keyed by action, e.g. `{ click: "#more" }`. `df.jsInstructions()` lists them. */
     jsInstructions?: unknown;
     blockResource?: string;
     /** Markdown only: always render just the `<main>` / `<article>` container. */
@@ -198,6 +200,9 @@ interface CrawlStatus {
     total_cost: number;
     /** Whether the crawl reached a final state. */
     done: boolean;
+    /** RFC 3339. */
+    created_at?: string;
+    updated_at?: string;
 }
 /** One page of crawl results. */
 interface CrawlResultsPage {
@@ -220,6 +225,11 @@ interface JobStatus {
     total_cost: number;
     done: boolean;
 }
+/** A cancelled job or crawl: its final progress and what was refunded. */
+interface CancelResult extends JobStatus {
+    refunded_tasks: number;
+    refunded_credits: number;
+}
 /** Every task of a job. A job completes even when some of its tasks failed. */
 interface JobResults {
     id: string;
@@ -228,15 +238,228 @@ interface JobResults {
     tasks_complete: number;
     tasks: Result[];
 }
+/** A task type, as the list and analytics filters take it. */
+type TaskType = "unlocker" | "llm_scraping" | "serp" | "map" | "crawl" | (string & {});
+/** Filters shared by {@link DataFuel.listJobs} and {@link DataFuel.listTasks}. */
+interface ListOptions {
+    status?: Status;
+    type?: TaskType;
+    /** Created on or after this day (UTC). A Date is sent as its UTC day. */
+    startDate?: string | Date;
+    /** Created on or before this day (UTC, inclusive). */
+    endDate?: string | Date;
+    /** Items per page. API default 50, max 200. */
+    limit?: number;
+    /** `nextCursor` of the previous page. */
+    cursor?: string;
+}
+/** {@link ListOptions} plus the job or crawl the tasks belong to. */
+interface ListTasksOptions extends ListOptions {
+    jobId?: string;
+}
+/** A job or crawl in a list, with the same counters as {@link JobStatus}. */
+interface JobSummary extends JobStatus {
+    id: string;
+    /** `crawl` for a crawl, otherwise the task type of the batch. */
+    type: TaskType;
+    /** RFC 3339. */
+    created_at: string;
+    updated_at: string;
+}
+/** One page of jobs, newest first. */
+interface JobsPage {
+    jobs: JobSummary[];
+    /** Absent on the last page. */
+    nextCursor?: string;
+}
+/** A task in a list. It has no result: fetch that with {@link DataFuel.getTask}. */
+interface TaskSummary {
+    id: string;
+    /** `null` for a task created on its own rather than by a job or crawl. */
+    job_id: string | null;
+    type: TaskType;
+    status: Status;
+    /** Absent for `llm_scraping` and `serp`. */
+    url?: string;
+    /** Charged when queued; a failed task is refunded. */
+    credit_cost: number;
+    /** RFC 3339. */
+    created_at: string;
+    processed_at?: string;
+    failed_at?: string;
+}
+/** One page of tasks, newest first. */
+interface TasksPage {
+    tasks: TaskSummary[];
+    /** Absent on the last page. */
+    nextCursor?: string;
+}
+/** What moved credits on the account. */
+type TransactionOperation = "plan_assignment" | "purchase" | "usage" | "refund" | "topup" | "expiry" | "adjustment" | (string & {});
+/** Filters and paging for {@link DataFuel.transactions}. */
+interface TransactionsOptions {
+    operation?: TransactionOperation;
+    startDate?: string | Date;
+    endDate?: string | Date;
+    /** 1-based. API default 1. */
+    page?: number;
+    /** API default 10, max 200. */
+    limit?: number;
+}
+/** One credit movement. `amount` is negative for usage and expiry. */
+interface Transaction {
+    id: number;
+    amount: number;
+    /** The part of `amount` that moved plan credits, same sign; the rest moved pay-as-you-go credits. */
+    plan_amount?: number;
+    operation: TransactionOperation;
+    /** What `reference_id` points to, e.g. `task_id` or `job_id`. */
+    reference_type: string;
+    reference_id: string;
+    /** The balance right after this movement. */
+    balance_after?: number;
+    /** RFC 3339. */
+    created_at: string;
+}
+/** Total of one operation over the whole filtered range, not just the page. */
+interface TransactionSum {
+    operation: TransactionOperation;
+    total: number;
+    count: number;
+}
+/** One page of credit movements, newest first. */
+interface TransactionsPage {
+    transactions: Transaction[];
+    /** Movements matching the filters across all pages. */
+    total_count: number;
+    sums: TransactionSum[];
+}
+/** Range and grouping for {@link DataFuel.analytics}. */
+interface AnalyticsOptions {
+    /** API default 30 days ago. The range may span at most 365 days. */
+    startDate?: string | Date;
+    /** Inclusive. API default today. */
+    endDate?: string | Date;
+    /** Time series bucket. API default `daily`. */
+    interval?: "hourly" | "daily" | "weekly" | "monthly";
+    /** Restrict to one task type. */
+    module?: TaskType;
+}
+/** Task counts and net credits of one slice. Failed tasks are refunded and count 0 credits. */
+interface AnalyticsCounts {
+    total: number;
+    completed: number;
+    failed: number;
+    credits_used: number;
+}
+/** Tasks by the HTTP status the target answered. `status_code` 0 means no answer (timeout, DNS). */
+interface StatusCodeBreakdown {
+    status_code: number;
+    count: number;
+    completed: number;
+    failed: number;
+    credits_used: number;
+    avg_credits_per_request: number;
+}
+/** The same figures for the equally long period before, and the change in percent. */
+interface PreviousPeriod {
+    credits_used: number;
+    fulfilled_requests: number;
+    failed_requests: number;
+    failed_percentage: number;
+    efficiency_score: number;
+    credits_used_change: number;
+    fulfilled_requests_change: number;
+    failed_requests_change: number;
+    efficiency_score_change: number;
+}
+/** Usage over a date range. Percentages are 0-100. */
+interface Analytics {
+    summary: {
+        total_tasks: number;
+        credits_used: number;
+        fulfilled_requests: number;
+        failed_requests: number;
+        failed_percentage: number;
+        success_rate: number;
+        efficiency_score: number;
+        avg_credits_per_request: number;
+        avg_duration_ms: number;
+        previous_period?: PreviousPeriod | null;
+    };
+    timeseries: (AnalyticsCounts & {
+        period: string;
+    })[];
+    by_module: (AnalyticsCounts & {
+        module: string;
+        success_rate: number;
+        avg_credits_per_request: number;
+        avg_duration_ms: number;
+        status_codes: StatusCodeBreakdown[];
+    })[];
+    top_targets: (AnalyticsCounts & {
+        target: string;
+    })[];
+    by_status_code: StatusCodeBreakdown[];
+}
+/**
+ * Credits by pool. Plan credits are spent first, roll over when the plan renews and
+ * expire if it is not renewed. Pay-as-you-go credits come from credit packs, are spent
+ * after plan credits and never expire.
+ */
+interface BalanceSplit {
+    /** Total spendable credits: `plan_balance` plus `payg_balance`. */
+    balance: number;
+    plan_balance: number;
+    payg_balance: number;
+}
 /** The account behind the API key. */
 interface Profile {
     email: string;
     username: string;
     current_concurrency: number;
     concurrency_limit: number;
+    /** Total spendable credits: `plan_credit_balance` plus `payg_credit_balance`. */
     credit_balance: number;
+    /** Spent first; expire if the plan is not renewed. */
+    plan_credit_balance?: number;
+    /** Spent after plan credits; never expire. */
+    payg_credit_balance?: number;
     monthly_credit_limit: number;
 }
+/** One argument of a browser action. */
+interface JsInstructionArg {
+    name: string;
+    type: string;
+    values?: string[];
+    required: boolean;
+}
+/** One browser action `jsInstructions` accepts. */
+interface JsInstruction {
+    action: string;
+    description: string;
+    /** Shape of the value: scalar, array or object. */
+    value: string;
+    args: JsInstructionArg[];
+    /** Whether it can target an element inside an iframe. */
+    iframe: boolean;
+    example: unknown;
+}
+/** A named proxy location: a city, or an ASN. */
+interface ProxyLocation {
+    code: string;
+    name: string;
+}
+/** A proxy country with its regions and their cities. */
+interface ProxyCountry {
+    code: string;
+    name: string;
+    regions: {
+        code: string;
+        name: string;
+        cities: ProxyLocation[];
+    }[];
+}
 /** One task type or LLM engine, and whether it accepts new work. */
 interface Capability {
     name: string;
@@ -265,13 +488,14 @@ declare class Capabilities {
 
 declare const DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
 /** Kept in step with package.json by a test; see test/hardening.test.ts. */
-declare const VERSION = "0.1.0";
+declare const VERSION = "0.3.0";
 /** Options for the llm_scraping module, which reads its proxy country here. */
 interface AskOptions {
     engine: Engine | string;
     websearch?: boolean;
     followUp?: string;
     country?: string;
+    location?: string;
     format?: string;
 }
 /** Options for `map`. */
@@ -303,6 +527,40 @@ interface CrawlOptions extends ScrapeOptions {
     /** Pages in flight, default 5. */
     concurrency?: number;
 }
+/** Options for `search`, the Google SERP module. Proxy type is not used. */
+interface SearchOptions {
+    /** Google `gl`, e.g. "us". */
+    country?: string;
+    /** Google `hl`, e.g. "en". */
+    language?: string;
+    /** Canonical location name. Give at most one of location, uule, lat/lon. */
+    location?: string;
+    /** 1-based, default 1. */
+    page?: number;
+    /** e.g. "google.de". */
+    googleDomain?: string;
+    uule?: string;
+    lat?: number;
+    lon?: number;
+    /** Metres around lat/lon or location, max 1000. */
+    radius?: number;
+    cr?: string;
+    lr?: string;
+    tbs?: string;
+    safe?: "active" | "off";
+    nfpr?: boolean;
+    filter?: boolean;
+    uds?: string;
+    kgmid?: string;
+    si?: string;
+    ludocid?: string;
+    lsig?: string;
+    ibp?: string;
+    /** Exit country of the request. */
+    proxyCountry?: string;
+    /** Default json. */
+    format?: "json" | "html" | "markdown";
+}
 
 /** The client. It only moves bytes; `core` decides what goes on the wire. */
 
@@ -332,7 +590,8 @@ interface ClientOptions {
  *
  * Pick the call by the shape of the work: one URL is {@link scrape}, a site's
  * URL list is {@link map}, many pages from a start URL is {@link crawl}, a list
- * of known URLs is {@link runJob}, a question for an AI engine is {@link ask}.
+ * of known URLs is {@link runJob}, a question for an AI engine is {@link ask}, a
+ * Google search is {@link search}.
  *
  * Every write carries an `Idempotency-Key`, generated per request, so a retry
  * attaches to the task already running instead of charging twice.
@@ -375,6 +634,8 @@ declare class DataFuel {
     getTask(taskId: string, options?: CallOptions): Promise<Result>;
     /** Send a prompt to an AI engine and return its answer. */
     ask(prompt: string, options: AskOptions & CallOptions): Promise<Result>;
+    /** Run a Google search and return the results page, parsed to JSON by default. */
+    search(query: string, options?: SearchOptions & CallOptions): Promise<Result>;
     /**
      * List the URLs of a site without scraping them.
      *
@@ -390,6 +651,11 @@ declare class DataFuel {
      * refunded. Unset limits use the API defaults: 100 pages, depth 3, 5 in flight.
      */
     startCrawl(url: string, options?: CrawlOptions & CallOptions): Promise<string>;
+    /**
+     * Stop a crawl. Queued pages are refunded, pages in flight finish and bill.
+     * Throws {@link JobNotCancellable} when it already finished.
+     */
+    cancelCrawl(crawlId: string, options?: CallOptions): Promise<CancelResult>;
     /** Return the progress of a crawl. */
     getCrawl(crawlId: string, options?: CallOptions): Promise<CrawlStatus>;
     /** One page of results, in discovery order. `limit` unset uses the API default. */
@@ -430,8 +696,17 @@ declare class DataFuel {
     createAskJob(prompts: string[], options: AskOptions & CallOptions & {
         sequential?: boolean;
     }): Promise<string>;
+    /** Queue a batch of Google searches and return the job id. */
+    createSearchJob(queries: string[], options?: SearchOptions & CallOptions & {
+        sequential?: boolean;
+    }): Promise<string>;
     /** Return the progress of a job. */
     getJob(jobId: string, options?: CallOptions): Promise<JobStatus>;
+    /**
+     * Stop a job. Queued tasks are refunded, tasks in flight finish and bill.
+     * Throws {@link JobNotCancellable} when it already finished.
+     */
+    cancelJob(jobId: string, options?: CallOptions): Promise<CancelResult>;
     /**
      * Return the per-task results of a job.
      *
@@ -439,6 +714,13 @@ declare class DataFuel {
      * `task.raiseForStatus()` per task.
      */
     jobResults(jobId: string, options?: CallOptions): Promise<JobResults>;
+    /** One page of your jobs and crawls, newest first. Pass `nextCursor` back as `cursor`. */
+    listJobs(options?: ListOptions & CallOptions): Promise<JobsPage>;
+    /**
+     * One page of your tasks, newest first, including those of jobs and crawls.
+     * Items carry no result: fetch it with {@link getTask}.
+     */
+    listTasks(options?: ListTasksOptions & CallOptions): Promise<TasksPage>;
     /** Poll until the job is done. Without `timeoutMs` it waits indefinitely. */
     waitJob(jobId: string, options?: CallOptions): Promise<JobStatus>;
     /** Create a job, wait for it, and return its results. */
@@ -449,10 +731,33 @@ declare class DataFuel {
     runAskJob(prompts: string[], options: AskOptions & CallOptions & {
         sequential?: boolean;
     }): Promise<JobResults>;
-    /** Which task types and LLM engines are switched on right now. */
+    /** Create a search job, wait for it, and return its results. */
+    runSearchJob(queries: string[], options?: SearchOptions & CallOptions & {
+        sequential?: boolean;
+    }): Promise<JobResults>;
+    /** Which task types and LLM engines are switched on right now. Needs no key. */
     capabilities(options?: CallOptions): Promise<Capabilities>;
-    /** Remaining credits. */
+    /** The browser actions `jsInstructions` accepts, with their arguments. Needs no key. */
+    jsInstructions(options?: CallOptions): Promise<JsInstruction[]>;
+    /** Countries, regions and cities a proxy type can exit from. */
+    proxyLocations(options?: CallOptions & {
+        proxyType?: string;
+    }): Promise<ProxyCountry[]>;
+    /** ASNs a proxy type can exit from in one country (ISO 3166-1 alpha-2). */
+    proxyAsns(country: string, options?: CallOptions & {
+        proxyType?: string;
+    }): Promise<ProxyLocation[]>;
+    /** Remaining credits: plan and pay-as-you-go together. */
     balance(options?: CallOptions): Promise<number>;
+    /** Remaining credits by pool: plan credits (spent first) and pay-as-you-go credits. */
+    balanceSplit(options?: CallOptions): Promise<BalanceSplit>;
+    /**
+     * Credit movements, newest first: plan assignments, credit pack purchases, usage, refunds, expiry.
+     * `sums` totals each operation over the whole range, not just this page.
+     */
+    transactions(options?: TransactionsOptions & CallOptions): Promise<TransactionsPage>;
+    /** Usage over a date range: totals, a time series and breakdowns. Default: the last 30 days. */
+    analytics(options?: AnalyticsOptions & CallOptions): Promise<Analytics>;
     /** The account behind the API key. */
     me(options?: CallOptions): Promise<Profile>;
     /**
@@ -486,10 +791,12 @@ declare class NoApiKey extends DataFuelError {
 /** The request never got an answer: DNS, connection, abort, read timeout. */
 declare class TransportError extends DataFuelError {
 }
+/** The `code` of an API error. The API may add codes; unknown ones pass through. */
+type ErrorCode = "UNAUTHORIZED" | "INVALID_API_KEY" | "FORBIDDEN" | "INSUFFICIENT_CREDITS" | "RATE_LIMIT_EXCEEDED" | "CONCURRENCY_LIMIT_REACHED" | "INVALID_REQUEST_BODY" | "INVALID_ATTRIBUTES" | "MISSING_TARGET" | "UNSUPPORTED_TASK_TYPE" | "JOB_REQUIRES_MULTIPLE_TARGETS" | "INVALID_IDEMPOTENCY_KEY" | "IDEMPOTENCY_KEY_REUSED" | "INVALID_TASK_ID" | "INVALID_JOB_ID" | "TASK_NOT_FOUND" | "JOB_NOT_FOUND" | "CRAWL_NOT_FOUND" | "JOB_NOT_CANCELLABLE" | "INVALID_CRAWL_PATTERN" | "CRAWL_UNSUPPORTED_OPTION" | "INVALID_CURSOR" | "INVALID_QUERY_PARAM" | "INVALID_PROXY_TYPE" | "INVALID_COUNTRY" | "INVALID_DATE_FORMAT" | "INVALID_DATE_RANGE" | "INVALID_INTERVAL" | "MODULE_UNAVAILABLE" | "ENGINE_UNAVAILABLE" | "API_KEY_RESET_FAILED" | "TASK_RESULT_TIMEOUT" | "INTERNAL_ERROR" | (string & {});
 /** A non-2xx answer from the API itself. */
 declare class APIError extends DataFuelError {
     readonly status: number;
-    readonly code: string;
+    readonly code: ErrorCode;
     /** Seconds the API asked us to wait, from Retry-After. 0 when absent. */
     readonly retryAfter: number;
     constructor(status: number, code: string, message: string, retryAfter?: number);
@@ -497,6 +804,9 @@ declare class APIError extends DataFuelError {
 /** 401: the API key is missing or invalid. */
 declare class Unauthorized extends APIError {
 }
+/** 403 FORBIDDEN: the account is inactive. */
+declare class Forbidden extends APIError {
+}
 /** 404: unknown id, or one that belongs to another account. */
 declare class NotFound extends APIError {
 }
@@ -512,7 +822,10 @@ declare class InvalidAttributes extends APIError {
 /** 422: the key was already used for a different request. */
 declare class IdempotencyKeyReused extends APIError {
 }
-/** 503: an operator switched something off. The message carries the reason. */
+/** 409 JOB_NOT_CANCELLABLE: the job or crawl already finished. */
+declare class JobNotCancellable extends APIError {
+}
+/** 503: an operator switched something off, or a dependency is down. The message carries the reason. */
 declare class Unavailable extends APIError {
 }
 /** 503 MODULE_UNAVAILABLE: this task type is switched off. Nothing was charged. */
@@ -551,4 +864,4 @@ declare class WaitTimeout extends DataFuelError {
     constructor(message: string, id?: string, status?: unknown);
 }
 
-export { type AI, APIError, type AskOptions, Blocked, type CallOptions, Capabilities, type Capability, type ClientOptions, type CrawlOptions, CrawlPage, type CrawlResult, type CrawlResultsPage, type CrawlStatus, DEFAULT_BASE_URL, DataFuel, DataFuelError, type Engine, EngineUnavailable, type Format, IdempotencyKeyReused, InsufficientCredits, InvalidAttributes, type JobResults, type JobStatus, type Link, type MapOptions, ModuleUnavailable, NoApiKey, NotFound, type Payload, type Profile, type Proxy, type ProxyType, RateLimited, Result, type ScrapeOptions, type SiteMap, type Status, TaskFailed, TransportError, Unauthorized, Unavailable, VERSION, WaitTimeout, isDone };
+export { type AI, APIError, type Analytics, type AnalyticsCounts, type AnalyticsOptions, type AskOptions, type BalanceSplit, Blocked, type CallOptions, type CancelResult, Capabilities, type Capability, type ClientOptions, type CrawlOptions, CrawlPage, type CrawlResult, type CrawlResultsPage, type CrawlStatus, DEFAULT_BASE_URL, DataFuel, DataFuelError, type Engine, EngineUnavailable, type ErrorCode, Forbidden, type Format, IdempotencyKeyReused, InsufficientCredits, InvalidAttributes, JobNotCancellable, type JobResults, type JobStatus, type JobSummary, type JobsPage, type JsInstruction, type JsInstructionArg, type Link, type ListOptions, type ListTasksOptions, type MapOptions, ModuleUnavailable, NoApiKey, NotFound, type Payload, type PreviousPeriod, type Profile, type Proxy, type ProxyCountry, type ProxyLocation, type ProxyType, RateLimited, Result, type ScrapeOptions, type SearchOptions, type SiteMap, type Status, type StatusCodeBreakdown, TaskFailed, type TaskSummary, type TaskType, type TasksPage, type Transaction, type TransactionOperation, type TransactionSum, type TransactionsOptions, type TransactionsPage, TransportError, Unauthorized, Unavailable, VERSION, WaitTimeout, isDone };