@datafuel/sdk 0.1.0

package/dist/index.js

@@ -0,0 +1,873 @@
+// src/errors.ts
+var DataFuelError = class extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = new.target.name;
+  }
+};
+var NoApiKey = class extends DataFuelError {
+};
+var TransportError = class extends DataFuelError {
+};
+var APIError = class extends DataFuelError {
+  status;
+  code;
+  /** Seconds the API asked us to wait, from Retry-After. 0 when absent. */
+  retryAfter;
+  constructor(status, code, message, retryAfter = 0) {
+    super(code ? `${message} (${status} ${code})` : `${message} (${status})`);
+    this.status = status;
+    this.code = code;
+    this.retryAfter = retryAfter;
+  }
+};
+var Unauthorized = class extends APIError {
+};
+var NotFound = class extends APIError {
+};
+var RateLimited = class extends APIError {
+};
+var InsufficientCredits = class extends APIError {
+};
+var InvalidAttributes = class extends APIError {
+};
+var IdempotencyKeyReused = class extends APIError {
+};
+var Unavailable = class extends APIError {
+};
+var ModuleUnavailable = class extends Unavailable {
+};
+var EngineUnavailable = class extends Unavailable {
+};
+var TaskFailed = class extends DataFuelError {
+  result;
+  constructor(result) {
+    const detail = result.error ?? result.payload?.error_detail ?? "no detail";
+    super(`task failed: ${detail}`);
+    this.result = result;
+  }
+  /** Anti-bot vendor recognised on the target, when there was one. */
+  get protection() {
+    return this.result.protection;
+  }
+};
+var Blocked = class extends TaskFailed {
+};
+var WaitTimeout = class extends DataFuelError {
+  id;
+  status;
+  constructor(message, id, status) {
+    super(message);
+    this.id = id;
+    this.status = status;
+  }
+};
+var BY_CODE = {
+  MODULE_UNAVAILABLE: ModuleUnavailable,
+  ENGINE_UNAVAILABLE: EngineUnavailable,
+  INSUFFICIENT_CREDITS: InsufficientCredits,
+  INVALID_ATTRIBUTES: InvalidAttributes,
+  IDEMPOTENCY_KEY_REUSED: IdempotencyKeyReused,
+  INVALID_API_KEY: Unauthorized
+};
+var BY_STATUS = {
+  401: Unauthorized,
+  402: InsufficientCredits,
+  404: NotFound,
+  422: IdempotencyKeyReused,
+  429: RateLimited,
+  503: Unavailable
+};
+function apiError(status, body, retryAfter = 0) {
+  let code = "";
+  let message = "";
+  if (body !== null && typeof body === "object") {
+    const record2 = body;
+    code = typeof record2.code === "string" ? record2.code : "";
+    message = typeof record2.message === "string" ? record2.message : "";
+  } else if (typeof body === "string" && body.length > 0 && body.length <= 300) {
+    message = body;
+  }
+  const Cls = BY_CODE[code] ?? BY_STATUS[status] ?? APIError;
+  return new Cls(status, code, message || `HTTP ${status}`, retryAfter);
+}
+
+// src/core.ts
+var DEFAULT_BASE_URL = "https://scraping-api.datafuel.ai/api/v1";
+var VERSION = "0.1.0";
+var DEFAULT_TIMEOUT_MS = 18e4;
+var STILL_PROCESSING_DELAY_MS = 2e3;
+var STILL_PROCESSING = "TASK_STILL_PROCESSING";
+var MAX_IDEMPOTENCY_KEY = 255;
+var Request = class {
+  constructor(method, path, params, body, idempotencyKey) {
+    this.method = method;
+    this.path = path;
+    this.params = params;
+    this.body = body;
+    this.idempotencyKey = idempotencyKey;
+  }
+  method;
+  path;
+  params;
+  body;
+  idempotencyKey;
+  /** GETs are safe by nature, writes because they carry an idempotency key. */
+  get retryable() {
+    return this.method === "GET" || this.idempotencyKey !== void 0;
+  }
+  /**
+   * Method and path only. The body can hold an LLM key or a cookie header, so
+   * neither logging nor JSON.stringify may spill it.
+   */
+  toString() {
+    return `<Request ${this.method} ${this.path}>`;
+  }
+  toJSON() {
+    return this.toString();
+  }
+  [/* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom")]() {
+    return this.toString();
+  }
+};
+function pathSegment(value) {
+  return encodeURIComponent(value);
+}
+function newIdempotencyKey() {
+  return crypto.randomUUID();
+}
+function headers(apiKey, userAgent, request) {
+  const out = {
+    "X-API-Key": apiKey,
+    Accept: "application/json",
+    "User-Agent": userAgent
+  };
+  if (request.body !== void 0) out["Content-Type"] = "application/json";
+  if (request.idempotencyKey !== void 0) out["Idempotency-Key"] = request.idempotencyKey;
+  return out;
+}
+function selector(value) {
+  return typeof value === "string" ? value : JSON.stringify(value);
+}
+function aiAttributes(ai) {
+  const out = { result_use_ai: true };
+  if (ai.prompt) out.result_ai_prompt = ai.prompt;
+  if (ai.format !== void 0) out.result_ai_format = ai.format;
+  if (ai.provider) out.ai_provider = ai.provider;
+  if (ai.model) out.ai_model = ai.model;
+  if (ai.apiKey) out.ai_api_key = ai.apiKey;
+  return out;
+}
+function validateAI(ai) {
+  const given = {
+    provider: ai.provider !== void 0,
+    model: ai.model !== void 0,
+    apiKey: ai.apiKey !== void 0
+  };
+  const present = Object.values(given).filter(Boolean).length;
+  if (present > 0 && present < 3) {
+    const missing = Object.entries(given).filter(([, ok]) => !ok).map(([name]) => name).join(", ");
+    throw new TypeError(`ai needs provider, model and apiKey together; missing: ${missing}`);
+  }
+}
+function scrapeAttributes(options = {}) {
+  const attrs = {};
+  if (options.format) attrs.result_format = options.format;
+  if (options.jsRendering) attrs.js_rendering = true;
+  if (options.waitFor) attrs.wait_for_selector = options.waitFor;
+  if (options.waitForTimeoutMs) attrs.wait_for_selector_timeout_ms = options.waitForTimeoutMs;
+  if (options.jsInstructions) attrs.js_instructions = options.jsInstructions;
+  if (options.blockResource) attrs.block_resource = options.blockResource;
+  if (options.mainContentOnly) attrs.main_content_only = true;
+  if (options.includeImages !== void 0) attrs.include_images = options.includeImages;
+  if (options.extract) attrs.extract_selector = selector(options.extract);
+  if (options.extractRegex) attrs.extract_regex = selector(options.extractRegex);
+  if (options.template) attrs.result_template = options.template;
+  if (options.method) attrs.method = options.method;
+  if (options.body) attrs.body = options.body;
+  if (options.contentType) attrs.content_type = options.contentType;
+  if (options.headers) attrs.headers = options.headers;
+  if (options.headerOrder) attrs.header_order = options.headerOrder;
+  if (options.cookies) attrs.cookie_string = options.cookies;
+  if (options.userAgent) attrs.user_agent = options.userAgent;
+  if (options.userAgentType) attrs.user_agent_type = options.userAgentType;
+  if (options.ai) {
+    validateAI(options.ai);
+    Object.assign(attrs, aiAttributes(options.ai));
+  }
+  return attrs;
+}
+function proxyEnvelope(proxy) {
+  const out = {};
+  if (!proxy) return out;
+  if (proxy.type) out.proxy_type = proxy.type;
+  if (proxy.country) out.proxy_country = proxy.country;
+  if (proxy.city) out.proxy_city = proxy.city;
+  if (proxy.state) out.proxy_state = proxy.state;
+  if (proxy.asn) out.proxy_asn = proxy.asn;
+  return out;
+}
+function proxySession(proxy) {
+  const out = {};
+  if (!proxy) return out;
+  if (proxy.sessionId) out.proxy_session_id = proxy.sessionId;
+  if (proxy.ttl) out.proxy_ttl = proxy.ttl;
+  return out;
+}
+function envelope(taskType, attributes, extra = {}) {
+  const body = { type: taskType, ...proxyEnvelope(extra.proxy) };
+  if (extra.multithreaded !== void 0) body.multithreaded = extra.multithreaded;
+  body.attributes = attributes;
+  return body;
+}
+function key(explicit) {
+  if (explicit !== void 0 && explicit.length > MAX_IDEMPOTENCY_KEY) {
+    throw new TypeError(`idempotencyKey is longer than ${MAX_IDEMPOTENCY_KEY} characters`);
+  }
+  return explicit ?? newIdempotencyKey();
+}
+function buildScrape(url, options, proxy, idempotencyKey) {
+  const attrs = { ...scrapeAttributes(options), url, ...proxySession(proxy) };
+  return new Request(
+    "POST",
+    "/task",
+    void 0,
+    envelope("unlocker", attrs, { proxy }),
+    idempotencyKey
+  );
+}
+function buildJob(urls, options, proxy, sequential, idempotencyKey) {
+  const attrs = { ...scrapeAttributes(options), urls: [...urls] };
+  return new Request(
+    "POST",
+    "/job",
+    void 0,
+    envelope("unlocker", attrs, { proxy, multithreaded: !sequential }),
+    idempotencyKey
+  );
+}
+function askAttributes(promptField, prompt, options) {
+  const attrs = { [promptField]: prompt, engine: options.engine };
+  if (options.websearch) attrs.websearch = true;
+  if (options.followUp) attrs.follow_up_prompt = options.followUp;
+  if (options.country) attrs.proxy_country = options.country;
+  if (options.format) attrs.result_format = options.format;
+  return attrs;
+}
+function buildAsk(prompt, options, idempotencyKey) {
+  const attrs = askAttributes("prompt", prompt, options);
+  return new Request("POST", "/task", void 0, envelope("llm_scraping", attrs), idempotencyKey);
+}
+function buildAskJob(prompts, options, sequential, idempotencyKey) {
+  const attrs = askAttributes("prompts", [...prompts], options);
+  return new Request(
+    "POST",
+    "/job",
+    void 0,
+    envelope("llm_scraping", attrs, { multithreaded: !sequential }),
+    idempotencyKey
+  );
+}
+function buildMap(url, options, proxy, idempotencyKey) {
+  const attrs = { url };
+  if (options.search) attrs.search = options.search;
+  if (options.sitemap) attrs.sitemap = options.sitemap;
+  if (options.limit) attrs.limit = options.limit;
+  if (options.includeSubdomains) attrs.include_subdomains = true;
+  if (options.ignoreSitemap) attrs.ignore_sitemap = true;
+  if (options.sitemapOnly) attrs.sitemap_only = true;
+  if (options.userAgent) attrs.user_agent = options.userAgent;
+  if (options.userAgentType) attrs.user_agent_type = options.userAgentType;
+  Object.assign(attrs, proxySession(proxy));
+  return new Request("POST", "/map", void 0, envelope("map", attrs, { proxy }), idempotencyKey);
+}
+function buildCrawl(url, options, proxy, idempotencyKey) {
+  if (options.ai) {
+    throw new TypeError("crawl does not support ai: the API rejects result_use_ai on crawls");
+  }
+  const attrs = { ...scrapeAttributes(options), url };
+  if (options.maxPages) attrs.max_pages = options.maxPages;
+  if (options.maxDepth) attrs.max_depth = options.maxDepth;
+  if (options.concurrency) attrs.concurrency = options.concurrency;
+  if (options.includePaths?.length) attrs.include_paths = [...options.includePaths];
+  if (options.excludePaths?.length) attrs.exclude_paths = [...options.excludePaths];
+  if (options.includeSubdomains) attrs.include_subdomains = true;
+  if (options.allowBackwardLinks) attrs.allow_backward_links = true;
+  return new Request(
+    "POST",
+    "/crawl",
+    void 0,
+    envelope("crawl", attrs, { proxy }),
+    idempotencyKey
+  );
+}
+function crawlResultsRequest(crawlId, cursor, limit) {
+  const params = {};
+  if (cursor) params.cursor = cursor;
+  if (limit) params.limit = String(limit);
+  return new Request(
+    "GET",
+    `/crawl/${pathSegment(crawlId)}/results`,
+    Object.keys(params).length > 0 ? params : void 0
+  );
+}
+function stillProcessing(body) {
+  return body !== null && typeof body === "object" && body.code === STILL_PROCESSING;
+}
+function shouldRetry(error) {
+  if (error instanceof Unavailable) {
+    if (error.code === "MODULE_UNAVAILABLE" || error.code === "ENGINE_UNAVAILABLE") return false;
+  }
+  if (error instanceof APIError) return [429, 502, 503, 504].includes(error.status);
+  return true;
+}
+function backoff(attempt, retryAfterSeconds = 0) {
+  if (retryAfterSeconds > 0) return Math.min(retryAfterSeconds * 1e3, 3e4);
+  const base = 500 * 2 ** Math.min(attempt, 4);
+  return base / 2 + Math.random() * (base / 2);
+}
+function pollDelay(attempt) {
+  return Math.max(STILL_PROCESSING_DELAY_MS, backoff(attempt));
+}
+
+// src/models.ts
+var TERMINAL = /* @__PURE__ */ new Set(["completed", "completed_with_errors", "failed", "cancelled"]);
+function isDone(status) {
+  return status !== void 0 && TERMINAL.has(status);
+}
+var Result = class {
+  id;
+  status;
+  /** HTTP status the target answered. */
+  statusCode;
+  finalUrl;
+  redirected;
+  /** Charged for this task, 0 when it failed. */
+  creditsUsed;
+  durationMs;
+  blocked;
+  /** Anti-bot vendor recognised when blocked. */
+  protection;
+  error;
+  /** The raw `result` object. Prefer `text`, `data` and `image`. */
+  payload;
+  /** Everything the API sent, including fields this SDK does not know yet. */
+  raw;
+  constructor(raw) {
+    this.raw = raw;
+    this.id = str(raw.id);
+    this.status = str(raw.status);
+    this.statusCode = num(raw.status_code);
+    this.finalUrl = str(raw.final_url);
+    this.redirected = raw.redirected === true;
+    this.creditsUsed = num(raw.credits_used) ?? 0;
+    this.durationMs = num(raw.duration_ms);
+    this.blocked = raw.blocked === true;
+    this.protection = str(raw.protection);
+    this.error = str(raw.error);
+    this.payload = raw.result !== null && typeof raw.result === "object" ? raw.result : void 0;
+  }
+  /** Effective state: the envelope's, else the payload's, else inferred. */
+  get state() {
+    if (this.status) return this.status;
+    if (this.payload?.status) return this.payload.status;
+    if (this.payload?.data !== void 0 && this.payload.data !== null) return "completed";
+    return "pending";
+  }
+  /** Whether the task has not finished yet (job and crawl results list stubs). */
+  get pending() {
+    return !isDone(this.state);
+  }
+  /** Whether the task completed and the target was not blocked. */
+  get ok() {
+    return this.state === "completed" && !this.blocked;
+  }
+  /** The structured output: format json, extract, template, AI. */
+  get data() {
+    return this.payload?.data;
+  }
+  /** Content of an html or markdown result; the raw JSON for structured ones. */
+  get text() {
+    const value = this.data;
+    if (value === void 0 || value === null) return "";
+    return typeof value === "string" ? value : JSON.stringify(value);
+  }
+  /** Bytes of a png or jpeg screenshot. Throws when the result is not one. */
+  get image() {
+    try {
+      return Uint8Array.from(atob(this.text), (char) => char.charCodeAt(0));
+    } catch (cause) {
+      throw new DataFuelError("result is not a png or jpeg screenshot", { cause });
+    }
+  }
+  /** `data`, or throw when the task carried none. */
+  requireData() {
+    if (this.data === void 0 || this.data === null) {
+      throw new DataFuelError(`result has no data (status ${this.state})`);
+    }
+    return this.data;
+  }
+  /** Throw TaskFailed, or Blocked, when this task failed. Refunded either way. */
+  raiseForStatus() {
+    if (this.state === "failed") {
+      throw this.blocked ? new Blocked(this) : new TaskFailed(this);
+    }
+  }
+};
+var CrawlPage = class extends Result {
+  url;
+  depth;
+  taskId;
+  constructor(raw) {
+    super(raw);
+    this.url = str(raw.url) ?? "";
+    this.depth = num(raw.depth) ?? 0;
+    this.taskId = str(raw.task_id);
+  }
+};
+var Capabilities = class {
+  modules;
+  engines;
+  constructor(raw) {
+    this.modules = Array.isArray(raw.modules) ? raw.modules : [];
+    this.engines = Array.isArray(raw.engines) ? raw.engines : [];
+  }
+  /** Whether a task type accepts work. Unknown names report false. */
+  moduleEnabled(name) {
+    return this.modules.some((item) => item.name === name && item.enabled);
+  }
+  /** Whether an LLM engine accepts work. Unknown names report false. */
+  engineEnabled(name) {
+    return this.engines.some((item) => item.name === name && item.enabled);
+  }
+};
+function str(value) {
+  return typeof value === "string" && value !== "" ? value : void 0;
+}
+function num(value) {
+  return typeof value === "number" ? value : void 0;
+}
+
+// src/client.ts
+var DataFuel = class {
+  baseUrl;
+  timeoutMs;
+  maxRetries;
+  pollIntervalMs;
+  userAgent;
+  apiKey;
+  fetchImpl;
+  constructor(options = {}) {
+    const opts = typeof options === "string" ? { apiKey: options } : options;
+    this.apiKey = opts.apiKey ?? envApiKey() ?? "";
+    this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+    this.timeoutMs = opts.timeoutMs === void 0 ? DEFAULT_TIMEOUT_MS : opts.timeoutMs;
+    this.maxRetries = Math.max(opts.maxRetries ?? 2, 0);
+    this.pollIntervalMs = opts.pollIntervalMs !== void 0 && opts.pollIntervalMs > 0 ? opts.pollIntervalMs : 2e3;
+    const sdk = `datafuel-js/${VERSION}`;
+    this.userAgent = opts.userAgent ? `${opts.userAgent} ${sdk}` : sdk;
+    this.fetchImpl = opts.fetch ?? globalThis.fetch.bind(globalThis);
+  }
+  // --- transport ---------------------------------------------------------
+  /** Pause between attempts. Overridable so tests need not wait in real time. */
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+  async send(request, opts = {}) {
+    if (!this.apiKey) {
+      throw new NoApiKey("no API key: pass one to the client or set DATAFUEL_API_KEY");
+    }
+    const url = new URL(this.baseUrl + request.path);
+    for (const [name, value] of Object.entries(request.params ?? {})) {
+      url.searchParams.set(name, value);
+    }
+    const init = {
+      method: request.method,
+      headers: headers(this.apiKey, this.userAgent, request),
+      // Redirects stay off: a redirect to another host would carry X-API-Key
+      // to it, since fetch does not strip custom headers.
+      redirect: "manual"
+    };
+    if (request.body !== void 0) init.body = JSON.stringify(request.body);
+    for (let attempt = 0; ; attempt++) {
+      let error;
+      try {
+        const signal = this.signalFor(opts);
+        const send = this.fetchImpl;
+        const response = await send(url, signal ? { ...init, signal } : init);
+        return await parse(response);
+      } catch (caught) {
+        if (isAbort(caught)) {
+          throw new TransportError("the request was aborted or timed out", { cause: caught });
+        }
+        error = caught instanceof DataFuelError ? caught : new TransportError(String(caught), {
+          cause: caught
+        });
+      }
+      const retryAfter = error instanceof Object && "retryAfter" in error ? Number(error.retryAfter) : 0;
+      if (attempt >= this.maxRetries || !request.retryable || !shouldRetry(error)) throw error;
+      await this.sleep(backoff(attempt, retryAfter));
+    }
+  }
+  signalFor(opts) {
+    const budget = opts.timeoutMs ?? this.timeoutMs;
+    const timeout = budget === null || budget === void 0 ? void 0 : AbortSignal.timeout(budget);
+    if (timeout && opts.signal) return AbortSignal.any([timeout, opts.signal]);
+    return timeout ?? opts.signal;
+  }
+  /**
+   * Send a task and wait for its result.
+   *
+   * On a 202 the API is still working: the identical request goes out again
+   * under the identical idempotency key, so it attaches to the running task
+   * rather than starting a second one.
+   */
+  async runTask(request, opts) {
+    const budget = opts.timeoutMs ?? this.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const deadline = Date.now() + budget;
+    let body;
+    for (let attempt = 0; ; attempt++) {
+      body = await this.send(request, opts);
+      if (!stillProcessing(body)) break;
+      const delay = pollDelay(attempt);
+      if (Date.now() + delay >= deadline) {
+        throw new WaitTimeout(
+          "the task is still processing; re-send under the same idempotency key to pick it up",
+          request.idempotencyKey
+        );
+      }
+      await this.sleep(delay);
+    }
+    const result = new Result(record(body));
+    result.raiseForStatus();
+    return result;
+  }
+  // --- pages -------------------------------------------------------------
+  /**
+   * Fetch one URL and wait for the result.
+   *
+   * Throws {@link Blocked} when the target refused the request and
+   * {@link TaskFailed} otherwise; both carry `.result`.
+   */
+  async scrape(url, options = {}) {
+    const request = buildScrape(url, options, options.proxy, key(options.idempotencyKey));
+    return this.runTask(request, options);
+  }
+  /** The one-liner: the page as LLM-ready markdown, images dropped. */
+  async markdown(url, options = {}) {
+    const result = await this.scrape(url, { ...options, format: "markdown", includeImages: false });
+    return result.text;
+  }
+  /**
+   * Return a task by id, e.g. one created by a job or a crawl.
+   *
+   * A task that is still running comes back with `pending` true.
+   */
+  async getTask(taskId, options = {}) {
+    const body = await this.send(
+      new Request("GET", `/task/${pathSegment(taskId)}`),
+      options
+    );
+    if (stillProcessing(body)) return new Result({ id: taskId, status: "processing" });
+    const result = new Result(record(body));
+    result.raiseForStatus();
+    return result;
+  }
+  /** Send a prompt to an AI engine and return its answer. */
+  async ask(prompt, options) {
+    const request = buildAsk(prompt, options, key(options.idempotencyKey));
+    return this.runTask(request, options);
+  }
+  /**
+   * List the URLs of a site without scraping them.
+   *
+   * One credit per call on a Basic proxy, however many links come back. Use it
+   * before a crawl to see how big a section is.
+   */
+  async map(url, options = {}) {
+    const request = buildMap(url, options, options.proxy, key(options.idempotencyKey));
+    const task = await this.runTask(request, options);
+    const data = record(task.requireData());
+    return { ...data, links: data.links ?? [], sitemaps: data.sitemaps ?? [], task };
+  }
+  // --- crawl -------------------------------------------------------------
+  /**
+   * Queue a crawl and return its id immediately.
+   *
+   * Follows links from the start URL and scrapes every page. Pages are charged
+   * like single scrapes when they are queued; failed and blocked pages are
+   * refunded. Unset limits use the API defaults: 100 pages, depth 3, 5 in flight.
+   */
+  async startCrawl(url, options = {}) {
+    const request = buildCrawl(url, options, options.proxy, key(options.idempotencyKey));
+    return strField(await this.send(request, options), "job_id");
+  }
+  /** Return the progress of a crawl. */
+  async getCrawl(crawlId, options = {}) {
+    const body = record(
+      await this.send(new Request("GET", `/crawl/${pathSegment(crawlId)}`), options)
+    );
+    const raw = body;
+    return {
+      ...raw,
+      status: raw.status,
+      // Counters the caller reads unconditionally must exist even on a thin answer.
+      pages: {
+        discovered: 0,
+        enqueued: 0,
+        done: 0,
+        failed: 0,
+        skipped: 0,
+        ...raw.pages ?? {}
+      },
+      depth_reached: raw.depth_reached ?? 0,
+      total_cost: raw.total_cost ?? 0,
+      done: isDone(raw.status)
+    };
+  }
+  /** One page of results, in discovery order. `limit` unset uses the API default. */
+  async crawlResults(crawlId, options = {}) {
+    const body = record(
+      await this.send(crawlResultsRequest(crawlId, options.cursor, options.limit), options)
+    );
+    const pages = Array.isArray(body.pages) ? body.pages : [];
+    const next = typeof body.next_cursor === "string" ? body.next_cursor : void 0;
+    return {
+      pages: pages.map((page) => new CrawlPage(record(page))),
+      ...next ? { nextCursor: next } : {}
+    };
+  }
+  /**
+   * Walk every page of a crawl, fetching result pages as needed.
+   *
+   * Readable while the crawl runs; unfinished pages report `pending`.
+   */
+  async *crawlPages(crawlId, options = {}) {
+    let cursor;
+    const seen = /* @__PURE__ */ new Set();
+    for (; ; ) {
+      const batch = await this.crawlResults(crawlId, { ...options, ...cursor ? { cursor } : {} });
+      yield* batch.pages;
+      cursor = batch.nextCursor;
+      if (!cursor) return;
+      if (seen.has(cursor)) {
+        throw new DataFuelError("the API repeated a crawl cursor; stopping to avoid a loop");
+      }
+      seen.add(cursor);
+    }
+  }
+  /** Poll until the crawl is done. Without `timeoutMs` it waits indefinitely. */
+  async waitCrawl(crawlId, options = {}) {
+    const { timeoutMs, ...perPoll } = options;
+    const deadline = timeoutMs === void 0 ? null : Date.now() + timeoutMs;
+    for (; ; ) {
+      const status = await this.getCrawl(crawlId, perPoll);
+      if (status.done) return status;
+      if (deadline !== null && Date.now() + this.pollIntervalMs >= deadline) {
+        throw new WaitTimeout(`crawl ${crawlId} is still running`, crawlId, status);
+      }
+      await this.sleep(this.pollIntervalMs);
+    }
+  }
+  /**
+   * Start a crawl, wait for it, and return every page.
+   *
+   * Check `stop_reason`: `insufficient_credits` means it ended early. For large
+   * crawls prefer {@link startCrawl} + {@link waitCrawl} + {@link crawlPages},
+   * which stream instead of holding every page in memory. On {@link WaitTimeout}
+   * the error carries the id: the crawl keeps running and billing.
+   */
+  async crawl(url, options = {}) {
+    const id = await this.startCrawl(url, options);
+    const status = await this.waitCrawl(id, options);
+    const pages = [];
+    for await (const page of this.crawlPages(id, options)) pages.push(page);
+    return { id, status, pages };
+  }
+  // --- jobs --------------------------------------------------------------
+  /**
+   * Queue a batch of known URLs and return the job id.
+   *
+   * Cheaper and more predictable than a crawl when you already have the URLs.
+   * `sequential` runs them one after the other; the default runs them
+   * concurrently, bounded by the account's concurrency limit.
+   */
+  async createJob(urls, options = {}) {
+    const request = buildJob(
+      urls,
+      options,
+      options.proxy,
+      options.sequential ?? false,
+      key(options.idempotencyKey)
+    );
+    return strField(await this.send(request, options), "id");
+  }
+  /** Queue a batch of prompts for an AI engine and return the job id. */
+  async createAskJob(prompts, options) {
+    const request = buildAskJob(
+      prompts,
+      options,
+      options.sequential ?? false,
+      key(options.idempotencyKey)
+    );
+    return strField(await this.send(request, options), "id");
+  }
+  /** Return the progress of a job. */
+  async getJob(jobId, options = {}) {
+    const body = record(
+      await this.send(new Request("GET", `/job/${pathSegment(jobId)}`), options)
+    );
+    const raw = body;
+    return {
+      status: raw.status,
+      tasks_count: raw.tasks_count ?? 0,
+      tasks_done: raw.tasks_done ?? 0,
+      tasks_remaining: raw.tasks_remaining ?? 0,
+      total_cost: raw.total_cost ?? 0,
+      done: isDone(raw.status)
+    };
+  }
+  /**
+   * Return the per-task results of a job.
+   *
+   * A job completes even when some of its tasks failed: check `task.ok` or call
+   * `task.raiseForStatus()` per task.
+   */
+  async jobResults(jobId, options = {}) {
+    const body = record(
+      await this.send(new Request("GET", `/job/${pathSegment(jobId)}/results`), options)
+    );
+    const tasks = Array.isArray(body.tasks_result) ? body.tasks_result : [];
+    return {
+      id: jobId,
+      tasks_count: Number(body.tasks_count ?? 0),
+      tasks_failed: Number(body.tasks_failed ?? 0),
+      tasks_complete: Number(body.tasks_complete ?? 0),
+      tasks: tasks.map((task) => new Result(record(task)))
+    };
+  }
+  /** Poll until the job is done. Without `timeoutMs` it waits indefinitely. */
+  async waitJob(jobId, options = {}) {
+    const { timeoutMs, ...perPoll } = options;
+    const deadline = timeoutMs === void 0 ? null : Date.now() + timeoutMs;
+    for (; ; ) {
+      const status = await this.getJob(jobId, perPoll);
+      if (status.done) return status;
+      if (deadline !== null && Date.now() + this.pollIntervalMs >= deadline) {
+        throw new WaitTimeout(`job ${jobId} is still running`, jobId, status);
+      }
+      await this.sleep(this.pollIntervalMs);
+    }
+  }
+  /** Create a job, wait for it, and return its results. */
+  async runJob(urls, options = {}) {
+    const id = await this.createJob(urls, options);
+    await this.waitJob(id, options);
+    return this.jobResults(id, options);
+  }
+  /** Create a prompt job, wait for it, and return its results. */
+  async runAskJob(prompts, options) {
+    const id = await this.createAskJob(prompts, options);
+    await this.waitJob(id, options);
+    return this.jobResults(id, options);
+  }
+  // --- account -----------------------------------------------------------
+  /** Which task types and LLM engines are switched on right now. */
+  async capabilities(options = {}) {
+    return new Capabilities(
+      record(await this.send(new Request("GET", "/capabilities"), options))
+    );
+  }
+  /** Remaining credits. */
+  async balance(options = {}) {
+    const body = await this.send(new Request("GET", "/users/@me/balance"), options);
+    return intField(body, "balance");
+  }
+  /** The account behind the API key. */
+  async me(options = {}) {
+    const body = await this.send(new Request("GET", "/users/@me"), options);
+    return record(body);
+  }
+  /**
+   * Revoke the current key and return the new one.
+   *
+   * This is the only time the new key is shown. The client keeps using the
+   * revoked one: store the new key and build a new client with it.
+   */
+  async resetApiKey(options = {}) {
+    const body = await this.send(new Request("POST", "/users/@me/api-key/reset"), options);
+    return strField(body, "api_key");
+  }
+};
+function envApiKey() {
+  return typeof process !== "undefined" ? process.env?.DATAFUEL_API_KEY : void 0;
+}
+async function parse(response) {
+  const text = await response.text();
+  let body;
+  if (text.length > 0) {
+    try {
+      body = JSON.parse(text);
+    } catch {
+      body = text;
+    }
+  }
+  if (!response.ok) {
+    const header = response.headers.get("Retry-After");
+    const retryAfter = header !== null && !Number.isNaN(Number(header)) ? Number(header) : 0;
+    throw apiError(response.status, body, retryAfter);
+  }
+  return body;
+}
+function isAbort(error) {
+  return error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError");
+}
+function record(body) {
+  if (body === null || typeof body !== "object" || Array.isArray(body)) {
+    throw new DataFuelError(`unexpected answer from the API: ${JSON.stringify(body) ?? "empty"}`);
+  }
+  return body;
+}
+function field(body, name) {
+  const value = record(body)[name];
+  if (value === void 0 || value === null) {
+    throw new DataFuelError(`unexpected answer from the API: no ${name} field`);
+  }
+  return value;
+}
+function strField(body, name) {
+  return String(field(body, name));
+}
+function intField(body, name) {
+  const value = Number(field(body, name));
+  if (Number.isNaN(value)) {
+    throw new DataFuelError(`unexpected answer from the API: ${name} is not a number`);
+  }
+  return value;
+}
+export {
+  APIError,
+  Blocked,
+  Capabilities,
+  CrawlPage,
+  DEFAULT_BASE_URL,
+  DataFuel,
+  DataFuelError,
+  EngineUnavailable,
+  IdempotencyKeyReused,
+  InsufficientCredits,
+  InvalidAttributes,
+  ModuleUnavailable,
+  NoApiKey,
+  NotFound,
+  RateLimited,
+  Result,
+  TaskFailed,
+  TransportError,
+  Unauthorized,
+  Unavailable,
+  VERSION,
+  WaitTimeout,
+  isDone
+};
+//# sourceMappingURL=index.js.map