@datafuel/sdk 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,554 @@
+/** Request options and response shapes. */
+/** State of a task, job or crawl. Unknown states are treated as not final. */
+type Status = "created" | "pending" | "processing" | "completed" | "completed_with_errors" | "failed" | "cancelled";
+/** Whether a state is final. */
+declare function isDone(status: string | undefined): boolean;
+/**
+ * Shape of the scraped content.
+ *
+ * `html` raw page (API default), `markdown` cleaned text (best for LLMs),
+ * `json` schema.org / JSON-LD, `png` / `jpeg` full-page screenshot.
+ */
+type Format = "html" | "markdown" | "json" | "png" | "jpeg";
+/** AI assistant `ask` can query. Engines can be switched off at runtime. */
+type Engine = "openai" | "gemini" | "google_ai_mode" | "perplexity" | "copilot";
+/** Proxy plan. Matched case-insensitively; deployments may define others. */
+type ProxyType = "Basic" | "Premium" | (string & {});
+/** The exit the request leaves from. Every field optional: the account default. */
+interface Proxy {
+    type?: ProxyType;
+    /** ISO 3166-1 alpha-2, e.g. "US". */
+    country?: string;
+    city?: string;
+    state?: string;
+    asn?: string;
+    /**
+     * Sticky session: the same exit across requests, `ttl` in seconds. Read by
+     * `scrape` and `map` only, and sent in attributes rather than the envelope.
+     */
+    sessionId?: string;
+    ttl?: number;
+}
+/**
+ * Post-process the page with an LLM, using your own key.
+ *
+ * Not supported on crawls: the API rejects `result_use_ai` there.
+ */
+interface AI {
+    /** What to extract. */
+    prompt?: string;
+    /** Example JSON object the output must follow. */
+    format?: unknown;
+    provider?: "openai" | "anthropic" | "google";
+    model?: string;
+    apiKey?: string;
+}
+/** Page options, shared by `scrape`, jobs and crawls. */
+interface ScrapeOptions {
+    format?: Format;
+    /**
+     * Render the page in a real browser: slower, five times the credits on a
+     * Basic proxy. Leave it off unless the page comes back empty without it.
+     */
+    jsRendering?: boolean;
+    waitFor?: string;
+    waitForTimeoutMs?: number;
+    jsInstructions?: unknown;
+    blockResource?: string;
+    /** Markdown only: always render just the `<main>` / `<article>` container. */
+    mainContentOnly?: boolean;
+    /** Markdown only: `false` drops images and saves tokens. */
+    includeImages?: boolean;
+    /**
+     * Extract only matching elements: field name to CSS selector. Append
+     * ` @attr` to read an attribute, e.g. `{ links: "a @href" }`.
+     */
+    extract?: string | Record<string, string>;
+    /** The same with RE2 patterns on the raw HTML. */
+    extractRegex?: string | Record<string, string>;
+    /**
+     * Built-in extractors, comma separated: images, links, headings,
+     * phone_numbers, emails, meta_tags, tables, schema_org, all.
+     */
+    template?: string;
+    method?: string;
+    body?: string;
+    contentType?: string;
+    headers?: Record<string, string>;
+    headerOrder?: string[];
+    cookies?: string;
+    userAgent?: string;
+    /** chrome, firefox, safari, edge. */
+    userAgentType?: string;
+    ai?: AI;
+}
+/** Options every call accepts. */
+interface CallOptions {
+    proxy?: Proxy;
+    /** Generated per request when omitted. Max 255 characters. */
+    idempotencyKey?: string;
+    /** Overrides the client's timeout for this call. */
+    timeoutMs?: number;
+    signal?: AbortSignal;
+}
+/** The raw `result` object of a task. */
+interface Payload {
+    data?: unknown;
+    status?: Status;
+    error_detail?: string;
+    status_code?: number;
+}
+/**
+ * One scraped target with its metadata.
+ *
+ * Read the metadata before the content: a 200 can be an error page,
+ * `redirected` means `finalUrl` is not what you asked for, and a 404 still
+ * completes and bills.
+ */
+declare class Result {
+    readonly id: string | undefined;
+    readonly status: Status | undefined;
+    /** HTTP status the target answered. */
+    readonly statusCode: number | undefined;
+    readonly finalUrl: string | undefined;
+    readonly redirected: boolean;
+    /** Charged for this task, 0 when it failed. */
+    readonly creditsUsed: number;
+    readonly durationMs: number | undefined;
+    readonly blocked: boolean;
+    /** Anti-bot vendor recognised when blocked. */
+    readonly protection: string | undefined;
+    readonly error: string | undefined;
+    /** The raw `result` object. Prefer `text`, `data` and `image`. */
+    readonly payload: Payload | undefined;
+    /** Everything the API sent, including fields this SDK does not know yet. */
+    readonly raw: Record<string, unknown>;
+    constructor(raw: Record<string, unknown>);
+    /** Effective state: the envelope's, else the payload's, else inferred. */
+    get state(): string;
+    /** Whether the task has not finished yet (job and crawl results list stubs). */
+    get pending(): boolean;
+    /** Whether the task completed and the target was not blocked. */
+    get ok(): boolean;
+    /** The structured output: format json, extract, template, AI. */
+    get data(): unknown;
+    /** Content of an html or markdown result; the raw JSON for structured ones. */
+    get text(): string;
+    /** Bytes of a png or jpeg screenshot. Throws when the result is not one. */
+    get image(): Uint8Array;
+    /** `data`, or throw when the task carried none. */
+    requireData(): unknown;
+    /** Throw TaskFailed, or Blocked, when this task failed. Refunded either way. */
+    raiseForStatus(): void;
+}
+/** One crawled page: where it was found, plus the scrape result. */
+declare class CrawlPage extends Result {
+    readonly url: string;
+    readonly depth: number;
+    readonly taskId: string | undefined;
+    constructor(raw: Record<string, unknown>);
+}
+/** One discovered URL. */
+interface Link {
+    url: string;
+    /** "sitemap" or "page". */
+    source: string;
+    /** Page links only. */
+    title?: string;
+    /** Sitemap links only. */
+    lastmod?: string;
+}
+/**
+ * What `map` found.
+ *
+ * `reason` says why `links` is empty: page_blocked, page_error,
+ * page_unreachable, no_links_on_page (try scrape with jsRendering), no_sitemap,
+ * sitemap_no_entries, search_no_match.
+ */
+interface SiteMap {
+    url: string;
+    links: Link[];
+    total: number;
+    /** `limit` or the time budget cut the list. */
+    truncated: boolean;
+    sitemaps: string[];
+    page_status_code: number;
+    reason?: string;
+    credits: number;
+    /** Envelope of the underlying task. */
+    task: Result;
+}
+/** Progress of a crawl. */
+interface CrawlStatus {
+    status: Status;
+    /** Empty while the crawl still grows: max_pages, max_depth_exhausted, insufficient_credits, cancelled. */
+    stop_reason?: string;
+    pages: {
+        discovered: number;
+        enqueued: number;
+        done: number;
+        failed: number;
+        skipped: number;
+    };
+    depth_reached: number;
+    /**
+     * Charged at queue time; refunds of failed pages are not subtracted. Sum
+     * `creditsUsed` over the pages for the net figure.
+     */
+    total_cost: number;
+    /** Whether the crawl reached a final state. */
+    done: boolean;
+}
+/** One page of crawl results. */
+interface CrawlResultsPage {
+    pages: CrawlPage[];
+    /** Absent on the last page. */
+    nextCursor?: string;
+}
+/** A finished crawl: why it stopped and what it found. */
+interface CrawlResult {
+    id: string;
+    status: CrawlStatus;
+    pages: CrawlPage[];
+}
+/** Progress of a job. */
+interface JobStatus {
+    status: Status;
+    tasks_count: number;
+    tasks_done: number;
+    tasks_remaining: number;
+    total_cost: number;
+    done: boolean;
+}
+/** Every task of a job. A job completes even when some of its tasks failed. */
+interface JobResults {
+    id: string;
+    tasks_count: number;
+    tasks_failed: number;
+    tasks_complete: number;
+    tasks: Result[];
+}
+/** The account behind the API key. */
+interface Profile {
+    email: string;
+    username: string;
+    current_concurrency: number;
+    concurrency_limit: number;
+    credit_balance: number;
+    monthly_credit_limit: number;
+}
+/** One task type or LLM engine, and whether it accepts new work. */
+interface Capability {
+    name: string;
+    enabled: boolean;
+    /** Operator's note when switched off. */
+    reason?: string;
+}
+/** What the API accepts right now. */
+declare class Capabilities {
+    readonly modules: Capability[];
+    readonly engines: Capability[];
+    constructor(raw: Record<string, unknown>);
+    /** Whether a task type accepts work. Unknown names report false. */
+    moduleEnabled(name: string): boolean;
+    /** Whether an LLM engine accepts work. Unknown names report false. */
+    engineEnabled(name: Engine | string): boolean;
+}
+
+/**
+ * Request building and answer classification. Pure functions, no I/O.
+ *
+ * Every decision lives here — which envelope a module needs, where the sticky
+ * session keys go, how `ai` expands, whether an answer is retryable — so the
+ * client only moves bytes.
+ */
+
+declare const DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
+/** Kept in step with package.json by a test; see test/hardening.test.ts. */
+declare const VERSION = "0.1.0";
+/** Options for the llm_scraping module, which reads its proxy country here. */
+interface AskOptions {
+    engine: Engine | string;
+    websearch?: boolean;
+    followUp?: string;
+    country?: string;
+    format?: string;
+}
+/** Options for `map`. */
+interface MapOptions {
+    /** Keep only links whose URL or title contains this. */
+    search?: string;
+    /** Map only this sitemap, from a previous `SiteMap.sitemaps`. */
+    sitemap?: string;
+    /** Default 5000, capped at 10000. */
+    limit?: number;
+    includeSubdomains?: boolean;
+    ignoreSitemap?: boolean;
+    sitemapOnly?: boolean;
+    userAgent?: string;
+    userAgentType?: string;
+}
+/** Options for `crawl`, on top of the page options applied to every page. */
+interface CrawlOptions extends ScrapeOptions {
+    /** Default 100, max 10000. The hard budget of the crawl. */
+    maxPages?: number;
+    /** Default 3, max 10. The start URL is depth 0. */
+    maxDepth?: number;
+    /** RE2 on path?query; when set only matches are followed. */
+    includePaths?: string[];
+    /** RE2 on path?query; exclude wins. */
+    excludePaths?: string[];
+    includeSubdomains?: boolean;
+    allowBackwardLinks?: boolean;
+    /** Pages in flight, default 5. */
+    concurrency?: number;
+}
+
+/** The client. It only moves bytes; `core` decides what goes on the wire. */
+
+/** How to build the client. */
+interface ClientOptions {
+    /** Falls back to `process.env.DATAFUEL_API_KEY`. */
+    apiKey?: string;
+    baseUrl?: string;
+    /** Milliseconds. `null` disables the bound; scrape blocks until the page is ready. */
+    timeoutMs?: number | null;
+    /** Retries after a network error or a 429/502/503/504 answer. Default 2. */
+    maxRetries?: number;
+    /** How often `waitJob` and `waitCrawl` poll. Default 2000ms. */
+    pollIntervalMs?: number;
+    /** Prefixes the SDK User-Agent with your application's. */
+    userAgent?: string;
+    /** Swap the fetch implementation, e.g. in tests. */
+    fetch?: typeof globalThis.fetch;
+}
+/**
+ * Client for the DataFuel scraping API.
+ *
+ * ```ts
+ * const df = new DataFuel();                       // reads DATAFUEL_API_KEY
+ * const markdown = await df.markdown("https://example.com");
+ * ```
+ *
+ * Pick the call by the shape of the work: one URL is {@link scrape}, a site's
+ * URL list is {@link map}, many pages from a start URL is {@link crawl}, a list
+ * of known URLs is {@link runJob}, a question for an AI engine is {@link ask}.
+ *
+ * Every write carries an `Idempotency-Key`, generated per request, so a retry
+ * attaches to the task already running instead of charging twice.
+ */
+declare class DataFuel {
+    readonly baseUrl: string;
+    readonly timeoutMs: number | null;
+    readonly maxRetries: number;
+    readonly pollIntervalMs: number;
+    readonly userAgent: string;
+    private readonly apiKey;
+    private readonly fetchImpl;
+    constructor(options?: ClientOptions | string);
+    /** Pause between attempts. Overridable so tests need not wait in real time. */
+    protected sleep(ms: number): Promise<void>;
+    private send;
+    private signalFor;
+    /**
+     * Send a task and wait for its result.
+     *
+     * On a 202 the API is still working: the identical request goes out again
+     * under the identical idempotency key, so it attaches to the running task
+     * rather than starting a second one.
+     */
+    private runTask;
+    /**
+     * Fetch one URL and wait for the result.
+     *
+     * Throws {@link Blocked} when the target refused the request and
+     * {@link TaskFailed} otherwise; both carry `.result`.
+     */
+    scrape(url: string, options?: ScrapeOptions & CallOptions): Promise<Result>;
+    /** The one-liner: the page as LLM-ready markdown, images dropped. */
+    markdown(url: string, options?: ScrapeOptions & CallOptions): Promise<string>;
+    /**
+     * Return a task by id, e.g. one created by a job or a crawl.
+     *
+     * A task that is still running comes back with `pending` true.
+     */
+    getTask(taskId: string, options?: CallOptions): Promise<Result>;
+    /** Send a prompt to an AI engine and return its answer. */
+    ask(prompt: string, options: AskOptions & CallOptions): Promise<Result>;
+    /**
+     * List the URLs of a site without scraping them.
+     *
+     * One credit per call on a Basic proxy, however many links come back. Use it
+     * before a crawl to see how big a section is.
+     */
+    map(url: string, options?: MapOptions & CallOptions): Promise<SiteMap>;
+    /**
+     * Queue a crawl and return its id immediately.
+     *
+     * Follows links from the start URL and scrapes every page. Pages are charged
+     * like single scrapes when they are queued; failed and blocked pages are
+     * refunded. Unset limits use the API defaults: 100 pages, depth 3, 5 in flight.
+     */
+    startCrawl(url: string, options?: CrawlOptions & CallOptions): Promise<string>;
+    /** Return the progress of a crawl. */
+    getCrawl(crawlId: string, options?: CallOptions): Promise<CrawlStatus>;
+    /** One page of results, in discovery order. `limit` unset uses the API default. */
+    crawlResults(crawlId: string, options?: CallOptions & {
+        cursor?: string;
+        limit?: number;
+    }): Promise<CrawlResultsPage>;
+    /**
+     * Walk every page of a crawl, fetching result pages as needed.
+     *
+     * Readable while the crawl runs; unfinished pages report `pending`.
+     */
+    crawlPages(crawlId: string, options?: CallOptions & {
+        limit?: number;
+    }): AsyncGenerator<CrawlPage>;
+    /** Poll until the crawl is done. Without `timeoutMs` it waits indefinitely. */
+    waitCrawl(crawlId: string, options?: CallOptions): Promise<CrawlStatus>;
+    /**
+     * Start a crawl, wait for it, and return every page.
+     *
+     * Check `stop_reason`: `insufficient_credits` means it ended early. For large
+     * crawls prefer {@link startCrawl} + {@link waitCrawl} + {@link crawlPages},
+     * which stream instead of holding every page in memory. On {@link WaitTimeout}
+     * the error carries the id: the crawl keeps running and billing.
+     */
+    crawl(url: string, options?: CrawlOptions & CallOptions): Promise<CrawlResult>;
+    /**
+     * Queue a batch of known URLs and return the job id.
+     *
+     * Cheaper and more predictable than a crawl when you already have the URLs.
+     * `sequential` runs them one after the other; the default runs them
+     * concurrently, bounded by the account's concurrency limit.
+     */
+    createJob(urls: string[], options?: ScrapeOptions & CallOptions & {
+        sequential?: boolean;
+    }): Promise<string>;
+    /** Queue a batch of prompts for an AI engine and return the job id. */
+    createAskJob(prompts: string[], options: AskOptions & CallOptions & {
+        sequential?: boolean;
+    }): Promise<string>;
+    /** Return the progress of a job. */
+    getJob(jobId: string, options?: CallOptions): Promise<JobStatus>;
+    /**
+     * Return the per-task results of a job.
+     *
+     * A job completes even when some of its tasks failed: check `task.ok` or call
+     * `task.raiseForStatus()` per task.
+     */
+    jobResults(jobId: string, options?: CallOptions): Promise<JobResults>;
+    /** Poll until the job is done. Without `timeoutMs` it waits indefinitely. */
+    waitJob(jobId: string, options?: CallOptions): Promise<JobStatus>;
+    /** Create a job, wait for it, and return its results. */
+    runJob(urls: string[], options?: ScrapeOptions & CallOptions & {
+        sequential?: boolean;
+    }): Promise<JobResults>;
+    /** Create a prompt job, wait for it, and return its results. */
+    runAskJob(prompts: string[], options: AskOptions & CallOptions & {
+        sequential?: boolean;
+    }): Promise<JobResults>;
+    /** Which task types and LLM engines are switched on right now. */
+    capabilities(options?: CallOptions): Promise<Capabilities>;
+    /** Remaining credits. */
+    balance(options?: CallOptions): Promise<number>;
+    /** The account behind the API key. */
+    me(options?: CallOptions): Promise<Profile>;
+    /**
+     * Revoke the current key and return the new one.
+     *
+     * This is the only time the new key is shown. The client keeps using the
+     * revoked one: store the new key and build a new client with it.
+     */
+    resetApiKey(options?: CallOptions): Promise<string>;
+}
+
+/**
+ * Errors this package throws.
+ *
+ * Two families, as in the Go and Python SDKs:
+ *
+ * - {@link APIError} and its subclasses — the API refused the request.
+ * - {@link TaskFailed} / {@link Blocked} — the API accepted the task but the
+ *   page could not be scraped. Failed tasks are refunded.
+ */
+
+/** Base class for every error this package throws. */
+declare class DataFuelError extends Error {
+    constructor(message: string, options?: {
+        cause?: unknown;
+    });
+}
+/** No key was passed and DATAFUEL_API_KEY is empty. Thrown before any request. */
+declare class NoApiKey extends DataFuelError {
+}
+/** The request never got an answer: DNS, connection, abort, read timeout. */
+declare class TransportError extends DataFuelError {
+}
+/** A non-2xx answer from the API itself. */
+declare class APIError extends DataFuelError {
+    readonly status: number;
+    readonly code: string;
+    /** Seconds the API asked us to wait, from Retry-After. 0 when absent. */
+    readonly retryAfter: number;
+    constructor(status: number, code: string, message: string, retryAfter?: number);
+}
+/** 401: the API key is missing or invalid. */
+declare class Unauthorized extends APIError {
+}
+/** 404: unknown id, or one that belongs to another account. */
+declare class NotFound extends APIError {
+}
+/** 429: the account's request rate or concurrency limit was reached. */
+declare class RateLimited extends APIError {
+}
+/** 402: not enough credits for this task. */
+declare class InsufficientCredits extends APIError {
+}
+/** 400 INVALID_ATTRIBUTES: the attributes do not match the task type. */
+declare class InvalidAttributes extends APIError {
+}
+/** 422: the key was already used for a different request. */
+declare class IdempotencyKeyReused extends APIError {
+}
+/** 503: an operator switched something off. The message carries the reason. */
+declare class Unavailable extends APIError {
+}
+/** 503 MODULE_UNAVAILABLE: this task type is switched off. Nothing was charged. */
+declare class ModuleUnavailable extends Unavailable {
+}
+/** 503 ENGINE_UNAVAILABLE: this LLM engine is switched off. Nothing was charged. */
+declare class EngineUnavailable extends Unavailable {
+}
+/**
+ * The task was accepted but could not be completed. Refunded.
+ *
+ * `result` carries the envelope: `statusCode`, `blocked`, `protection`, `error`.
+ */
+declare class TaskFailed extends DataFuelError {
+    readonly result: Result;
+    constructor(result: Result);
+    /** Anti-bot vendor recognised on the target, when there was one. */
+    get protection(): string | undefined;
+}
+/**
+ * The target refused or challenged the request (403/429/503, anti-bot wall).
+ *
+ * Refunded. Retry with `jsRendering: true` or a Premium proxy.
+ */
+declare class Blocked extends TaskFailed {
+}
+/**
+ * A wait ran out of time. The work keeps running and billing server-side.
+ *
+ * `id` picks it back up (`waitCrawl`, `jobResults`, or a re-send under the same
+ * idempotency key); `status` is the last one seen, when there was one.
+ */
+declare class WaitTimeout extends DataFuelError {
+    readonly id: string | undefined;
+    readonly status: unknown;
+    constructor(message: string, id?: string, status?: unknown);
+}
+
+export { type AI, APIError, type AskOptions, Blocked, type CallOptions, Capabilities, type Capability, type ClientOptions, type CrawlOptions, CrawlPage, type CrawlResult, type CrawlResultsPage, type CrawlStatus, DEFAULT_BASE_URL, DataFuel, DataFuelError, type Engine, EngineUnavailable, type Format, IdempotencyKeyReused, InsufficientCredits, InvalidAttributes, type JobResults, type JobStatus, type Link, type MapOptions, ModuleUnavailable, NoApiKey, NotFound, type Payload, type Profile, type Proxy, type ProxyType, RateLimited, Result, type ScrapeOptions, type SiteMap, type Status, TaskFailed, TransportError, Unauthorized, Unavailable, VERSION, WaitTimeout, isDone };