@octocrawl/sdk 0.3.0

package/dist/index.js

@@ -0,0 +1,1240 @@
+// packages/contracts/dist/extractor.js
+var QuoteState;
+(function(QuoteState2) {
+  QuoteState2["Present"] = "present";
+  QuoteState2["AbsentObserved"] = "absent_observed";
+  QuoteState2["Unobserved"] = "unobserved";
+  QuoteState2["Conflicting"] = "conflicting";
+})(QuoteState || (QuoteState = {}));
+
+// packages/contracts/dist/policy.js
+var DEFAULT_NETWORK_POLICY = {
+  origin: "request",
+  privateAllowlist: [],
+  maxRedirects: 5,
+  maxBodyBytes: 10 * 1024 * 1024,
+  maxDecompressedBytes: 50 * 1024 * 1024,
+  perHostConcurrency: 2,
+  perHostMinDelayMs: 250,
+  respectRobotsTxt: true
+};
+
+// packages/contracts/dist/ssrf.js
+var V4 = {
+  loopback: [ip4("127.0.0.0"), 8],
+  rfc1918a: [ip4("10.0.0.0"), 8],
+  rfc1918b: [ip4("172.16.0.0"), 12],
+  rfc1918c: [ip4("192.168.0.0"), 16],
+  linkLocal: [ip4("169.254.0.0"), 16],
+  cgnat: [ip4("100.64.0.0"), 10],
+  multicast: [ip4("224.0.0.0"), 4],
+  unspecified: ip4("0.0.0.0"),
+  metadata: ip4("169.254.169.254"),
+  ecsMetadata: ip4("169.254.170.2")
+};
+var V6 = {
+  unspecified: 0n,
+  loopback: 1n,
+  linkLocal: [0xfe80n << 112n, 10],
+  ula: [0xfcn << 120n, 7],
+  multicast: [0xffn << 120n, 8],
+  metadata: 0xfd00ec20000000000000000000000254n
+};
+function ip4(text) {
+  const parsed = parseV4(text);
+  if (parsed === null)
+    throw new Error(`bad fixture ip ${text}`);
+  return parsed;
+}
+function parseV4(text) {
+  const parts = text.split(".");
+  if (parts.length !== 4)
+    return null;
+  let n = 0;
+  for (const part of parts) {
+    if (!/^\d{1,3}$/.test(part))
+      return null;
+    const octet = Number(part);
+    if (octet > 255)
+      return null;
+    n = (n << 8) + octet;
+  }
+  return n >>> 0;
+}
+
+// packages/contracts/dist/compliance.js
+var RESEARCH_UA_COMMENT = "compatible; w2l-research/0.1; +https://github.com/77777R7/Octocrawl; research benchmark, one request per page";
+var RESEARCH_USER_AGENT = `Mozilla/5.0 (${RESEARCH_UA_COMMENT})`;
+var MAX_CONTACT_LENGTH = 200;
+function contactIssue(contact) {
+  if (contact.length === 0 || contact.length > MAX_CONTACT_LENGTH)
+    return `must be 1 to ${MAX_CONTACT_LENGTH} characters`;
+  if (!/^[\x20-\x7e]+$/.test(contact))
+    return "must be printable ASCII";
+  if (/[()\\]/.test(contact))
+    return "must not contain parentheses or backslashes, which would end the User-Agent comment";
+  if (/(?:Chrome|Chromium)\/|HeadlessChrome/.test(contact))
+    return "must not name a browser product: research mode declares a bot";
+  return null;
+}
+var SEC_DECLARED_NAME = "W2L Research";
+function isSecHost(host) {
+  const name = host.toLowerCase().replace(/\.$/, "");
+  return name === "sec.gov" || name.endsWith(".sec.gov");
+}
+function researchUserAgent(contact = null, host = null) {
+  if (contact === null)
+    return RESEARCH_USER_AGENT;
+  const issue = contactIssue(contact);
+  if (issue !== null)
+    throw new Error(`W2L_CONTACT ${issue}.`);
+  if (host !== null && isSecHost(host))
+    return `${SEC_DECLARED_NAME} ${contact}`;
+  return `Mozilla/5.0 (${RESEARCH_UA_COMMENT}; contact: ${contact})`;
+}
+var CHROME_MAJOR_FLOOR = 128;
+function browserUserAgent(chromeMajor) {
+  return `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeMajor}.0.0.0 Safari/537.36`;
+}
+function browserClientHints(chromeMajor) {
+  return {
+    "sec-ch-ua": `"Chromium";v="${chromeMajor}", "Google Chrome";v="${chromeMajor}", "Not;A=Brand";v="24"`,
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"macOS"'
+  };
+}
+function mobileBrowserUserAgent(chromeMajor) {
+  return `Mozilla/5.0 (Linux; Android 14; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${chromeMajor}.0.0.0 Mobile Safari/537.36`;
+}
+function mobileBrowserClientHints(chromeMajor) {
+  return {
+    "sec-ch-ua": `"Chromium";v="${chromeMajor}", "Google Chrome";v="${chromeMajor}", "Not;A=Brand";v="24"`,
+    "sec-ch-ua-mobile": "?1",
+    "sec-ch-ua-platform": '"Android"'
+  };
+}
+var BROWSER_FINGERPRINT = {
+  locale: "en-US",
+  timezoneId: "America/Los_Angeles",
+  viewport: { width: 1280, height: 800 },
+  screen: { width: 1920, height: 1080 },
+  deviceScaleFactor: 2,
+  isMobile: false,
+  hasTouch: false
+};
+function modeIdentity(mode, chromeMajor = CHROME_MAJOR_FLOOR, contact = null, host = null, device = "desktop") {
+  const userAgent = device === "mobile" ? mobileBrowserUserAgent(chromeMajor) : browserUserAgent(chromeMajor);
+  const clientHints = device === "mobile" ? mobileBrowserClientHints(chromeMajor) : browserClientHints(chromeMajor);
+  switch (mode) {
+    case "research":
+      return { mode, userAgent: researchUserAgent(contact, host), clientHints: {}, respectsRobots: true, lane: "browser_local" };
+    case "standard":
+      return { mode, userAgent, clientHints, respectsRobots: true, lane: "browser_local", device };
+    case "authed":
+      return { mode, userAgent, clientHints, respectsRobots: true, lane: "browser_local_authed", device };
+    case "proxy":
+      return { mode, userAgent, clientHints, respectsRobots: true, lane: "browser_proxy", device };
+  }
+}
+var MODE_IDENTITIES = {
+  research: modeIdentity("research"),
+  standard: modeIdentity("standard"),
+  authed: modeIdentity("authed"),
+  proxy: modeIdentity("proxy")
+};
+
+// packages/contracts/dist/regexSafety.js
+function set(ranges) {
+  const sorted = [...ranges].sort((a, b) => a[0] - b[0]);
+  const merged = [];
+  for (const [low, high] of sorted) {
+    const previous = merged[merged.length - 1];
+    if (previous !== void 0 && low <= previous[1] + 1)
+      previous[1] = Math.max(previous[1], high);
+    else
+      merged.push([low, high]);
+  }
+  return { kind: "set", ranges: merged };
+}
+var DIGIT = set([[48, 57]]);
+var WORD = set([[48, 57], [65, 90], [95, 95], [97, 122]]);
+var SPACE = set([[9, 13], [32, 32], [160, 160], [5760, 5760], [8192, 8202], [8232, 8233], [8239, 8239], [8287, 8287], [12288, 12288], [65279, 65279]]);
+
+// packages/contracts/dist/file.js
+var DEFAULT_MAX_FILE_BYTES = 50 * 1024 * 1024;
+var MAX_FILE_BYTES_CEILING = 500 * 1024 * 1024;
+
+// packages/contracts/dist/api.js
+var DEFAULT_SCRAPE_TIMEOUT_MS = 3e5;
+var WEBHOOK_EVENTS = ["started", "page", "completed", "failed", "cancelled"];
+var MAX_WEBHOOK_HEADERS = 32;
+var MAX_WEBHOOK_METADATA_ENTRIES = 32;
+var MAX_WEBHOOK_METADATA_VALUE_LENGTH = 1e3;
+var DEFAULT_MAP_TIMEOUT_MS = 6e4;
+var WS_TOKEN_PROTOCOL_PREFIX = "w2l.token.";
+var API_ERROR_CODES = ["invalid_json", "invalid_request", "unsupported_parameter", "unsupported_format", "unauthorized", "not_found", "conflict", "internal_error"];
+function isApiErrorCode(value) {
+  return API_ERROR_CODES.includes(value);
+}
+var RATE_LIMITED_CODE = "rate_limited";
+var PAGE_KEYS = ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
+var ATTRIBUTION_KEYS = ["origin", "integration"];
+var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var CRAWL_SCOPE_KEYS = ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
+var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var BATCH_KEYS = ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var BATCH_APPEND_KEYS = ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", ...ATTRIBUTION_KEYS];
+var MAP_SCOPE_KEYS = ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
+var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", ...ATTRIBUTION_KEYS];
+var WEBHOOK_HEADERS_MESSAGE = `webhook.headers must be an object of at most ${MAX_WEBHOOK_HEADERS} string values`;
+var WEBHOOK_METADATA_MESSAGE = `webhook.metadata must be an object of at most ${MAX_WEBHOOK_METADATA_ENTRIES} string values of at most ${MAX_WEBHOOK_METADATA_VALUE_LENGTH} characters`;
+var WEBHOOK_EVENTS_MESSAGE = `webhook.events must be a non-empty array of ${WEBHOOK_EVENTS.join(", ")} without duplicates`;
+var SCHEMA_STRUCTURE = ["type", "properties", "required", "items", "additionalProperties", "enum", "const", "$ref", "$defs", "definitions", "anyOf", "oneOf"];
+var SCHEMA_NUMBERS = ["minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "multipleOf"];
+var SCHEMA_COUNTS = ["minLength", "maxLength", "minItems", "maxItems"];
+var SCHEMA_ANNOTATIONS = ["title", "description", "$comment", "default", "examples", "deprecated", "readOnly", "writeOnly", "format"];
+var SCHEMA_KEYS = /* @__PURE__ */ new Set([...SCHEMA_STRUCTURE, ...SCHEMA_NUMBERS, ...SCHEMA_COUNTS, "pattern", "uniqueItems", ...SCHEMA_ANNOTATIONS]);
+var STRING_FORMATS = ["markdown", "links", "json", "html", "rawHtml", "images", "tables", "screenshot"];
+var FORMAT_NAMES = [...STRING_FORMATS, "attributes", "list"];
+var MIN_SCREENSHOT_VIEWPORT = { width: 320, height: 240 };
+var SCREENSHOT_VIEWPORT_MESSAGE = `screenshot viewport must be {width, height} with integers within ${MIN_SCREENSHOT_VIEWPORT.width}..${BROWSER_FINGERPRINT.screen.width} by ${MIN_SCREENSHOT_VIEWPORT.height}..${BROWSER_FINGERPRINT.screen.height}`;
+
+// packages/contracts/dist/firecrawl.js
+var SHIM_ATTRIBUTION = ["origin", "integration"];
+var SHIM_MAP_KEYS = ["url", "search", "sitemap", "ignoreSitemap", "sitemapOnly", "includeSubdomains", "ignoreQueryParameters", "limit", "timeout", ...SHIM_ATTRIBUTION];
+
+// packages/contracts/dist/evidenceRecord.js
+var keysOf = () => (keys) => keys;
+var EVIDENCE_RECORD_KEYS = {
+  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"]),
+  redirectChain: keysOf()(["urls", "complete"]),
+  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"]),
+  outputSha256: keysOf()(["markdown", "json"]),
+  extractor: keysOf()(["name", "version", "commit"]),
+  fieldEvidence: keysOf()(["source", "locator"]),
+  artifact: keysOf()(["kind", "path", "sha256", "bytes", "contentType"]),
+  identity: keysOf()(["userAgent", "mode", "contact", "device", "requestHeaders"]),
+  pageActions: keysOf()(["steps", "scriptRan"]),
+  pageActionStep: keysOf()(["type", "outcome"]),
+  requestHeader: keysOf()(["name", "valueSha256"])
+};
+
+// packages/sdk/src/version.ts
+var SDK_VERSION = "0.3.0";
+
+// packages/sdk/src/watcher.ts
+var DEFAULT_WATCH_POLL_INTERVAL_MS = 2e3;
+var MIN_WATCH_POLL_INTERVAL_MS = 250;
+var POLL_FAILURE_LIMIT = 5;
+var FINISHED = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
+var STREAM_FRAMES = /* @__PURE__ */ new Set(["catchup", "document", "snapshot", "done", "error"]);
+function pageCursor(page) {
+  const json = JSON.stringify({ createdAt: page.createdAt, id: page.id });
+  const bytes = new TextEncoder().encode(json);
+  let binary = "";
+  for (const byte of bytes) binary += String.fromCharCode(byte);
+  return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+}
+function parseSseBlock(block) {
+  let event;
+  let id;
+  const data = [];
+  for (const line of block.split(/\r?\n/)) {
+    if (line.length === 0 || line.startsWith(":")) continue;
+    const colon = line.indexOf(":");
+    const field = colon === -1 ? line : line.slice(0, colon);
+    const value = colon === -1 ? "" : line.slice(colon + 1).replace(/^ /, "");
+    if (field === "event") event = value;
+    else if (field === "data") data.push(value);
+    else if (field === "id") id = value;
+  }
+  if (event === void 0 || data.length === 0) return null;
+  return frameOf(event, data.join("\n"), id);
+}
+function frameOf(type, json, cursor) {
+  if (!STREAM_FRAMES.has(type)) return null;
+  let payload;
+  try {
+    payload = JSON.parse(json);
+  } catch {
+    return null;
+  }
+  if (payload === null || typeof payload !== "object") return null;
+  if (type === "error") return { type, error: payload };
+  if (type === "document") return cursor === void 0 ? null : { type, data: payload, cursor };
+  return { type, data: payload };
+}
+function parseSocketFrame(data) {
+  const text = typeof data === "string" ? data : data instanceof ArrayBuffer ? new TextDecoder().decode(data) : null;
+  if (text === null) return null;
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return null;
+  }
+  if (parsed === null || typeof parsed !== "object") return null;
+  const frame = parsed;
+  if (typeof frame.type !== "string" || !STREAM_FRAMES.has(frame.type)) return null;
+  if (frame.type === "error") return frame.error !== null && typeof frame.error === "object" ? { type: "error", error: frame.error } : null;
+  if (frame.data === null || typeof frame.data !== "object") return null;
+  if (frame.type === "document") return typeof frame.cursor === "string" ? { type: "document", data: frame.data, cursor: frame.cursor } : null;
+  return { type: frame.type, data: frame.data };
+}
+function statusOf(error) {
+  return error !== null && typeof error === "object" && typeof error.status === "number" ? error.status : null;
+}
+function sleep(ms, signal) {
+  return new Promise((resolve, reject) => {
+    if (signal.aborted) {
+      reject(signal.reason);
+      return;
+    }
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", abort);
+      resolve();
+    }, Math.max(0, ms));
+    const abort = () => {
+      clearTimeout(timer);
+      reject(signal.reason);
+    };
+    signal.addEventListener("abort", abort, { once: true });
+  });
+}
+var JobWatcher = class extends EventTarget {
+  jobId;
+  kind;
+  /** Every document delivered so far, each step once, in arrival order. */
+  data = [];
+  /** The job's status as last reported; null before the first report. */
+  status = null;
+  /** The transport delivering events; null before one is open and after the watch ends. */
+  transport = null;
+  client;
+  transportOption;
+  pollIntervalMs;
+  socketConstructor;
+  controller = new AbortController();
+  seen = /* @__PURE__ */ new Set();
+  log = [];
+  waiters = [];
+  /** The cursor of the last document delivered: where the next transport, or a poll, continues from. */
+  cursor;
+  lastSnapshot = "";
+  finished = false;
+  closed = false;
+  timer;
+  constructor(client, jobId, options = {}) {
+    super();
+    if (typeof jobId !== "string" || jobId.length === 0) throw new TypeError("jobId must be a non-empty string");
+    const kind = options.kind ?? "crawl";
+    if (kind !== "crawl" && kind !== "batch") throw new TypeError("kind must be 'crawl' or 'batch'");
+    const transport = options.transport ?? "auto";
+    if (!["auto", "websocket", "sse", "poll"].includes(transport)) throw new TypeError("transport must be 'auto', 'websocket', 'sse' or 'poll'");
+    const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_WATCH_POLL_INTERVAL_MS;
+    if (!Number.isFinite(pollIntervalMs) || pollIntervalMs < MIN_WATCH_POLL_INTERVAL_MS) throw new TypeError(`pollIntervalMs must be at least ${MIN_WATCH_POLL_INTERVAL_MS}`);
+    if (options.timeoutMs !== void 0 && !(Number.isFinite(options.timeoutMs) && options.timeoutMs >= 0)) throw new TypeError("timeoutMs must be a finite number of milliseconds, 0 or more");
+    if (options.after !== void 0 && (typeof options.after !== "string" || options.after.length === 0)) throw new TypeError("after must be a non-empty cursor");
+    this.client = client;
+    this.jobId = jobId;
+    this.kind = kind;
+    this.transportOption = transport;
+    this.pollIntervalMs = pollIntervalMs;
+    this.socketConstructor = options.WebSocket === null ? void 0 : options.WebSocket ?? globalThis.WebSocket;
+    this.cursor = options.after;
+    if (options.signal !== void 0) {
+      if (options.signal.aborted) this.close();
+      else options.signal.addEventListener("abort", () => this.close(), { once: true });
+    }
+    if (options.timeoutMs !== void 0) {
+      const timeoutMs = options.timeoutMs;
+      this.timer = setTimeout(() => this.fail({ code: "watcher_timeout", message: `job ${jobId} did not finish within ${timeoutMs} ms (last status: ${this.status ?? "unknown"})` }), timeoutMs);
+    }
+    void this.run();
+  }
+  /** Stops watching: no further event, the job untouched. */
+  close() {
+    if (this.closed) return;
+    this.closed = true;
+    this.stop();
+  }
+  /** The events from the start, then as they arrive, until done, error or close. */
+  [Symbol.asyncIterator]() {
+    let index = 0;
+    return {
+      next: async () => {
+        for (; ; ) {
+          if (index < this.log.length) return { value: this.log[index++], done: false };
+          if (this.finished || this.closed) return { value: void 0, done: true };
+          await new Promise((resolve) => this.waiters.push(resolve));
+        }
+      },
+      return: async () => ({ value: void 0, done: true })
+    };
+  }
+  get stopped() {
+    return this.finished || this.closed;
+  }
+  async run() {
+    const order = this.transportOption === "auto" ? ["websocket", "sse", "poll"] : [this.transportOption];
+    for (const transport of order) {
+      if (this.stopped) return;
+      const outcome = transport === "websocket" ? await this.watchSocket() : transport === "sse" ? await this.watchEvents() : await this.poll();
+      if (outcome !== "fallback") return;
+    }
+    this.transport = null;
+    if (!this.stopped) this.fail({ code: "transport_unavailable", message: `no transport could watch ${this.kind} ${this.jobId} (tried ${order.join(", ")})` });
+  }
+  /** The stream route of this job, with the cursor to resume after when one is known. */
+  streamUrl(suffix) {
+    const url = new URL(`${this.client.baseUrl}/v1/${this.kind === "crawl" ? "crawl" : "batches"}/${encodeURIComponent(this.jobId)}/${suffix}`);
+    if (this.cursor !== void 0) url.searchParams.set("after", this.cursor);
+    if (suffix === "ws") url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
+    return url.href;
+  }
+  /** The WebSocket route; the token, when there is one, as the `w2l.token.<token>` subprotocol, since the WebSocket API sets no header. */
+  watchSocket() {
+    const Socket = this.socketConstructor;
+    if (Socket === void 0) return Promise.resolve("fallback");
+    let socket;
+    try {
+      socket = new Socket(this.streamUrl("ws"), this.client.token === void 0 || this.client.token.length === 0 ? void 0 : [`${WS_TOKEN_PROTOCOL_PREFIX}${this.client.token}`]);
+    } catch {
+      return Promise.resolve("fallback");
+    }
+    return new Promise((resolve) => {
+      let settled = false;
+      const settle = (outcome) => {
+        if (settled) return;
+        settled = true;
+        this.controller.signal.removeEventListener("abort", onAbort);
+        resolve(outcome);
+      };
+      const onAbort = () => {
+        try {
+          socket.close(1e3, "closed");
+        } catch {
+        }
+        ;
+        settle("closed");
+      };
+      this.controller.signal.addEventListener("abort", onAbort, { once: true });
+      let opened = false;
+      socket.addEventListener("open", () => {
+        opened = true;
+        if (!this.stopped) this.transport = "websocket";
+      });
+      socket.addEventListener("message", (event) => {
+        const frame = parseSocketFrame(event.data);
+        if (frame !== null) this.handle(frame);
+        if (this.stopped) {
+          try {
+            socket.close(1e3, "done");
+          } catch {
+          }
+          ;
+          settle(this.finished ? "done" : "closed");
+        }
+      });
+      socket.addEventListener("close", (event) => {
+        if (this.finished) {
+          settle("done");
+          return;
+        }
+        if (this.closed) {
+          settle("closed");
+          return;
+        }
+        if (event.code === 4404) {
+          this.fail({ code: "not_found", message: `${this.kind} ${this.jobId} not found` });
+          settle("fatal");
+          return;
+        }
+        if (event.code === 4400) {
+          this.fail({ code: "invalid_request", message: event.reason ?? "cursor is not one this API issued" });
+          settle("fatal");
+          return;
+        }
+        settle("fallback");
+      });
+      socket.addEventListener("error", () => {
+        if (opened) return;
+        try {
+          socket.close();
+        } catch {
+        }
+        ;
+        settle(this.closed ? "closed" : "fallback");
+      });
+    });
+  }
+  /** The server-sent events route: 404 leaves it to polling, 401/403 ends the watch, a stream that ends before done hands over from the last cursor. */
+  async watchEvents() {
+    const url = this.streamUrl("events");
+    let res;
+    try {
+      res = await this.client.fetch(url, { headers: this.client.headers({ accept: "text/event-stream" }), signal: this.controller.signal });
+    } catch {
+      return this.stopped ? "closed" : "fallback";
+    }
+    if (res.status === 404) return "fallback";
+    if (res.status === 401 || res.status === 403) {
+      this.fail({ code: "unauthorized", message: `GET ${new URL(url).pathname} answered ${res.status}` });
+      return "fatal";
+    }
+    if (!res.ok || res.body === null) return "fallback";
+    if (this.stopped) return "closed";
+    this.transport = "sse";
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    const dispatch = (block) => {
+      const frame = parseSseBlock(block);
+      if (frame !== null) this.handle(frame);
+    };
+    try {
+      for (; ; ) {
+        const { value, done } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        for (let boundary = buffer.search(/\r?\n\r?\n/); boundary !== -1; boundary = buffer.search(/\r?\n\r?\n/)) {
+          const block = buffer.slice(0, boundary);
+          buffer = buffer.slice(boundary).replace(/^\r?\n\r?\n/, "");
+          dispatch(block);
+          if (this.stopped) {
+            try {
+              await reader.cancel();
+            } catch {
+            }
+            ;
+            return this.finished ? "done" : "closed";
+          }
+        }
+      }
+      if (buffer.trim().length > 0) dispatch(buffer);
+    } catch {
+      if (this.stopped) return this.finished ? "done" : "closed";
+    }
+    if (this.finished) return "done";
+    return this.closed ? "closed" : "fallback";
+  }
+  /** The status and listing routes every pollIntervalMs: the documents since the last cursor, a snapshot, and done once the status is terminal and the last pages are read. */
+  async poll() {
+    this.transport = "poll";
+    const signal = this.controller.signal;
+    const cursors = { pages: this.cursor, errors: this.cursor, items: this.cursor };
+    const drain = async (key, list) => {
+      for (; ; ) {
+        const page = await list(cursors[key]);
+        for (const item of page.items) this.document(item, pageCursor(item));
+        const last = page.items[page.items.length - 1];
+        cursors[key] = page.nextCursor ?? (last === void 0 ? cursors[key] : pageCursor(last));
+        if (!page.hasMore) return;
+      }
+    };
+    let failures = 0;
+    for (; ; ) {
+      if (this.stopped) return "closed";
+      try {
+        const report = this.kind === "crawl" ? await this.client.getCrawl(this.jobId, { signal }) : await this.client.getBatch(this.jobId, { signal });
+        if (this.kind === "batch") {
+          await drain("items", (cursor) => this.client.getBatchItems(this.jobId, { limit: 50, ...cursor === void 0 ? {} : { cursor } }, { signal }));
+        } else {
+          await drain("pages", (cursor) => this.client.getCrawlPages(this.jobId, { limit: 100, includeDuplicates: true, ...cursor === void 0 ? {} : { cursor } }, { signal }));
+          await drain("errors", (cursor) => this.client.getCrawlErrors(this.jobId, { limit: 100, ...cursor === void 0 ? {} : { cursor } }, { signal }));
+        }
+        if (this.stopped) return "closed";
+        if (FINISHED.has(report.status)) {
+          this.done(report);
+          return "done";
+        }
+        this.snapshot(report);
+        failures = 0;
+      } catch (error) {
+        if (this.stopped) return "closed";
+        const status = statusOf(error);
+        if (status === 404) {
+          this.fail({ code: "not_found", message: `${this.kind} ${this.jobId} not found` });
+          return "fatal";
+        }
+        if (status === 401 || status === 403) {
+          this.fail({ code: "unauthorized", message: `polling ${this.kind} ${this.jobId} answered ${status}` });
+          return "fatal";
+        }
+        if (++failures >= POLL_FAILURE_LIMIT) {
+          this.fail({ code: "poll_failed", message: `polling ${this.kind} ${this.jobId} failed ${failures} times in a row: ${error instanceof Error ? error.message : String(error)}` });
+          return "fatal";
+        }
+      }
+      try {
+        await sleep(this.pollIntervalMs, signal);
+      } catch {
+        return "closed";
+      }
+    }
+  }
+  handle(frame) {
+    if (this.stopped) return;
+    if (frame.type === "document") this.document(frame.data, frame.cursor);
+    else if (frame.type === "done") this.done(frame.data);
+    else if (frame.type === "error") this.fail(frame.error);
+    else this.snapshot(frame.data);
+  }
+  document(page, cursor) {
+    if (this.stopped || this.seen.has(page.id)) return;
+    this.seen.add(page.id);
+    this.data.push(page);
+    this.cursor = cursor;
+    this.emit({ type: "document", data: page });
+  }
+  snapshot(report) {
+    if (this.stopped) return;
+    this.status = report.status;
+    const encoded = JSON.stringify(report);
+    if (encoded === this.lastSnapshot) return;
+    this.lastSnapshot = encoded;
+    this.emit({ type: "snapshot", data: report });
+  }
+  done(report) {
+    if (this.stopped) return;
+    this.status = report.status;
+    this.finished = true;
+    this.emit({ type: "done", data: report });
+    this.stop();
+  }
+  fail(error) {
+    if (this.stopped) return;
+    this.finished = true;
+    this.emit({ type: "error", error });
+    this.stop();
+  }
+  emit(event) {
+    this.log.push(event);
+    this.dispatchEvent(new CustomEvent(event.type, { detail: event.type === "error" ? event.error : event.data }));
+    this.wake();
+  }
+  stop() {
+    if (this.timer !== void 0) {
+      clearTimeout(this.timer);
+      this.timer = void 0;
+    }
+    this.transport = this.finished ? this.transport : null;
+    this.controller.abort();
+    this.wake();
+  }
+  wake() {
+    const waiting = this.waiters.splice(0);
+    for (const resume of waiting) resume();
+  }
+};
+
+// packages/sdk/src/client.ts
+function environmentToken() {
+  try {
+    const token = globalThis.process?.env?.["W2L_API_TOKEN"];
+    return token === void 0 || token.length === 0 ? void 0 : token;
+  } catch {
+    return void 0;
+  }
+}
+var SCRAPE_ANSWER_MARGIN_MS = 3e4;
+var MAP_ANSWER_MARGIN_MS = 5e3;
+function headersWait(ms) {
+  if (globalThis.process?.versions?.undici === void 0) return void 0;
+  return {
+    dispatch(options, handler) {
+      const global = globalThis;
+      const dispatcher = global[/* @__PURE__ */ Symbol.for("undici.globalDispatcher.2")] ?? global[/* @__PURE__ */ Symbol.for("undici.globalDispatcher.1")];
+      if (dispatcher === void 0) throw new Error("no global fetch dispatcher");
+      return dispatcher.dispatch({ ...options, headersTimeout: ms }, handler);
+    }
+  };
+}
+var SDK_ORIGIN = `js-sdk@${SDK_VERSION}`;
+var WaitTimeoutError = class extends Error {
+  constructor(taskId, last, timeoutMs, options) {
+    super(last === null ? `task ${taskId} status not read within ${timeoutMs} ms` : `task ${taskId} still ${last.status} after ${timeoutMs} ms`, options);
+    this.taskId = taskId;
+    this.last = last;
+    this.timeoutMs = timeoutMs;
+  }
+  taskId;
+  last;
+  timeoutMs;
+  name = "WaitTimeoutError";
+};
+var CRAWL_PAGE_MAX_LIMIT = 1e3;
+var BATCH_ITEM_MAX_LIMIT = 50;
+var BATCH_MAX_URLS = 1e3;
+function chunkUrls(urls, chunkSize = 100) {
+  if (!Number.isInteger(chunkSize) || chunkSize < 1 || chunkSize > BATCH_MAX_URLS) throw new RangeError(`chunkSize must be an integer between 1 and ${BATCH_MAX_URLS}`);
+  const chunks = [];
+  for (let start = 0; start < urls.length; start += chunkSize) chunks.push(urls.slice(start, start + chunkSize));
+  return chunks;
+}
+function hrefOf(url) {
+  try {
+    return new URL(url).href;
+  } catch {
+    return url;
+  }
+}
+function checkPaginationLimits(options) {
+  if (options.maxPages !== void 0 && !(Number.isInteger(options.maxPages) && options.maxPages >= 0)) throw new RangeError("maxPages must be an integer, 0 or more");
+  if (options.maxResults !== void 0 && !(Number.isInteger(options.maxResults) && options.maxResults >= 1)) throw new RangeError("maxResults must be an integer, 1 or more");
+  if (options.maxWaitMs !== void 0 && !(Number.isFinite(options.maxWaitMs) && options.maxWaitMs >= 0)) throw new RangeError("maxWaitMs must be a finite number of milliseconds, 0 or more");
+}
+var W2LError = class extends Error {
+  constructor(message, status, code, method, path, body, retryAfterMs2 = null, agentHints = []) {
+    super(message);
+    this.status = status;
+    this.code = code;
+    this.method = method;
+    this.path = path;
+    this.body = body;
+    this.retryAfterMs = retryAfterMs2;
+    this.agentHints = agentHints;
+  }
+  status;
+  code;
+  method;
+  path;
+  body;
+  retryAfterMs;
+  agentHints;
+  name = "W2LError";
+};
+async function responseError(method, path, res, message) {
+  const text = await res.text();
+  let body = text;
+  try {
+    body = JSON.parse(text);
+  } catch {
+  }
+  const fields = body !== null && typeof body === "object" ? body : {};
+  const code = isApiErrorCode(fields.code) || fields.code === RATE_LIMITED_CODE ? fields.code : void 0;
+  const agentHints = Array.isArray(fields.agentHints) ? fields.agentHints.filter((hint) => typeof hint === "string") : [];
+  return new W2LError(message ?? `${method} ${path} failed: ${res.status} ${text}`, res.status, code, method, path, body, retryAfterMs(res.headers.get("retry-after")), agentHints);
+}
+function retryAfterMs(value) {
+  if (value === null) return null;
+  const trimmed = value.trim();
+  if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1e3;
+  const at = /^[+-]?[\d.]+$/.test(trimmed) ? Number.NaN : Date.parse(trimmed);
+  return Number.isFinite(at) ? Math.max(0, at - Date.now()) : null;
+}
+var MAX_RETRY_AFTER_MS = 6e4;
+function retryDelayMs(error, failures) {
+  if (error instanceof W2LError) {
+    if (error.status !== 408 && error.status !== 429 && error.status < 500) return null;
+    if (error.retryAfterMs !== null) return error.retryAfterMs <= MAX_RETRY_AFTER_MS ? error.retryAfterMs : null;
+  } else if (!(error instanceof TypeError)) return null;
+  return Math.min(1e4, 1e3 * 2 ** (failures - 1));
+}
+function checkWaitOptions(options) {
+  for (const name of ["pollIntervalMs", "timeoutMs"]) {
+    const value = options[name];
+    if (value !== void 0 && !(Number.isFinite(value) && value >= 0)) throw new RangeError(`${name} must be a finite number of milliseconds, 0 or more`);
+  }
+  if (options.maxRetries !== void 0 && !(Number.isInteger(options.maxRetries) && options.maxRetries >= 0)) throw new RangeError("maxRetries must be an integer, 0 or more");
+}
+var FINISHED2 = ["completed", "failed", "cancelled"];
+var W2L = class {
+  baseUrl;
+  token;
+  fetchImpl;
+  /** The platform's fetch, not one passed in options, whose own limits are the caller's. */
+  platformFetch;
+  constructor(options) {
+    this.baseUrl = options.baseUrl.replace(/\/$/, "");
+    this.token = options.token ?? environmentToken();
+    this.fetchImpl = options.fetch ?? fetch;
+    this.platformFetch = options.fetch === void 0;
+  }
+  async scrape(url, opts = {}, request = {}) {
+    const deadlineMs = Number.isInteger(opts.timeout) ? Math.min(Math.max(opts.timeout, 0), DEFAULT_SCRAPE_TIMEOUT_MS) : DEFAULT_SCRAPE_TIMEOUT_MS;
+    const handedOver = opts.handoff !== void 0 && opts.handoff !== false;
+    return this.post("/v1/scrape", { ...opts, url, origin: originOf(opts, request) }, 200, request, handedOver ? 0 : deadlineMs + SCRAPE_ANSWER_MARGIN_MS);
+  }
+  /** The record of one scrape call, by the `scrapeId` its response carried (`metadata.scrapeId`); a W2LError with code `not_found` for an id the server has no record of. */
+  async getScrape(id, request = {}) {
+    return this.get(`/v1/scrapes/${encodeURIComponent(id)}`, request, `scrape not found: ${id}`);
+  }
+  /**
+   * The URLs of a site from its sitemaps and its start page's links, without
+   * fetching each page (POST /v1/map). The API answers by the map's deadline
+   * with what it found; the SDK waits that long plus MAP_ANSWER_MARGIN_MS.
+   */
+  async map(url, opts = {}, request = {}) {
+    const deadlineMs = Number.isInteger(opts.timeout) ? Math.max(opts.timeout, 0) : DEFAULT_MAP_TIMEOUT_MS;
+    return this.post("/v1/map", { ...opts, url, origin: originOf(opts, request) }, 200, request, deadlineMs + MAP_ANSWER_MARGIN_MS);
+  }
+  /** The record of one map, by the `id` its response carried; a W2LError with code `not_found` for an id the server has no record of. */
+  async getMap(id, request = {}) {
+    return this.get(`/v1/maps/${encodeURIComponent(id)}`, request, `map not found: ${id}`);
+  }
+  async crawl(url, opts = {}, request = {}) {
+    return this.post("/v1/crawl", { ...opts, url, origin: originOf(opts, request) }, 202, request);
+  }
+  /**
+   * Starts a batch: `{ taskId }`, plus `invalidURLs` (the entries skipped)
+   * when `ignoreInvalidURLs` was on; with `idempotencyKey` a retried start
+   * returns the first one's answer with `replayed: true`; with `appendToId`
+   * the URLs join that batch (see appendToBatch) and the answer carries
+   * `requested` and `appended`.
+   */
+  async batchScrape(urls, opts = {}, request = {}) {
+    return this.post("/v1/batches", { ...opts, urls, origin: originOf(opts, request) }, 202, request);
+  }
+  /**
+   * Adds URLs to an existing batch (`appendToId`): the job keeps its mode,
+   * formats, includeLinks, maxConcurrency and page options, and its run picks
+   * the URLs up (a completed batch runs again for them). The answer carries
+   * `requested`, the job's URLs now, and `appended`. A cancelled or failed
+   * batch, a total over 1000 or a URL already in the batch is a W2LError.
+   */
+  async appendToBatch(id, urls, opts = {}, request = {}) {
+    return this.batchScrape(urls, { ...opts, appendToId: id }, request);
+  }
+  /**
+   * Runs a list of any length as batches of `chunkSize` URLs (default 100),
+   * one after another: each job is started, waited for (as waitBatch, with
+   * the WaitOptions) and listed before the next starts. The items are merged
+   * in the order the URLs were submitted; the jobs stay on the server as
+   * ordinary batches, each with its own task directory. A caller's
+   * `idempotencyKey` becomes `<key>:<chunkIndex>` per job, so a retry of the
+   * whole call replays the jobs that went through. A WaitTimeoutError or
+   * W2LError from any job ends the call, naming that job; the earlier jobs
+   * are complete and the later chunks were never sent.
+   */
+  async batchScrapeChunked(urls, opts = {}, options = {}) {
+    if (opts.appendToId !== void 0) throw new TypeError("batchScrapeChunked cannot append; use appendToBatch");
+    const { chunkSize = 100, itemLimit = BATCH_ITEM_MAX_LIMIT, ...wait } = options;
+    checkWaitOptions(wait);
+    if (!Number.isInteger(itemLimit) || itemLimit < 1 || itemLimit > BATCH_ITEM_MAX_LIMIT) throw new RangeError(`itemLimit must be an integer between 1 and ${BATCH_ITEM_MAX_LIMIT}`);
+    const chunks = chunkUrls(urls, chunkSize);
+    const jobs = [];
+    const items = [];
+    const invalidURLs = [];
+    for (const [index, chunk] of chunks.entries()) {
+      const accepted = await this.batchScrape(chunk, { ...opts, ...opts.idempotencyKey === void 0 ? {} : { idempotencyKey: `${opts.idempotencyKey}:${index}` } }, wait);
+      const report = await this.waitBatch(accepted.taskId, wait);
+      const order = new Map(chunk.map((url, position) => [hrefOf(url), position]));
+      const listed = [];
+      for await (const item of this.listBatchItems(accepted.taskId, { limit: itemLimit }, wait)) listed.push(item);
+      items.push(...listed.sort((a, b) => (order.get(a.url) ?? Number.MAX_SAFE_INTEGER) - (order.get(b.url) ?? Number.MAX_SAFE_INTEGER)));
+      jobs.push({ taskId: accepted.taskId, urls: chunk.length, report, ...accepted.invalidURLs === void 0 ? {} : { invalidURLs: accepted.invalidURLs } });
+      if (accepted.invalidURLs !== void 0) invalidURLs.push(...accepted.invalidURLs);
+    }
+    return { jobs, items, invalidURLs };
+  }
+  async getBatch(id, request = {}) {
+    return this.get(`/v1/batches/${encodeURIComponent(id)}`, request, `batch not found: ${id}`);
+  }
+  /** The batch's failed, blocked, cancelled and budget-cut items across every attempt, in pages of up to 1000 (`limit`, `cursor`), with `robotsBlocked`, the URLs robots.txt refused. */
+  async getBatchErrors(id, options = {}, request = {}) {
+    const params = new URLSearchParams();
+    if (options.cursor !== void 0) params.set("cursor", options.cursor);
+    if (options.limit !== void 0) params.set("limit", String(options.limit));
+    const suffix = params.size === 0 ? "" : `?${params.toString()}`;
+    return this.get(`/v1/batches/${encodeURIComponent(id)}/errors${suffix}`, request, `batch not found: ${id}`);
+  }
+  async getBatchItems(id, options = {}, request = {}) {
+    return this.getPageList(`/v1/batches/${encodeURIComponent(id)}/items`, options, request);
+  }
+  /** Every item of a batch, page by page (at most 50 per request), or as many as the PaginationLimits allow; the generator's return value says where it stopped. */
+  listBatchItems(id, options = {}, request = {}) {
+    return this.paginate((query) => this.getBatchItems(id, query, request), options, request, BATCH_ITEM_MAX_LIMIT, "batch items");
+  }
+  /** The items listBatchItems would yield under the same options, collected, with where the listing stopped. */
+  async collectBatchItems(id, options = {}, request = {}) {
+    return collect(this.listBatchItems(id, options, request));
+  }
+  /** A batch's status and its items in one answer, every item unless the PaginationLimits stop the listing. */
+  async getBatchDocuments(id, options = {}, request = {}) {
+    const report = await this.getBatch(id, request);
+    const { items, nextCursor, stoppedBy } = await this.collectBatchItems(id, options, request);
+    return { report, items, nextCursor, stoppedBy };
+  }
+  /** Polls a batch until it completes, fails or is cancelled. Items come from listBatchItems. */
+  async waitBatch(id, options = {}) {
+    return this.waitFor(id, (request) => this.getBatch(id, request), options);
+  }
+  /** Starts a batch, waits for it (as waitBatch) and lists every item, failed ones included. */
+  async batchAndWait(urls, opts = {}, wait = {}) {
+    checkWaitOptions(wait);
+    const { taskId } = await this.batchScrape(urls, opts, wait);
+    const report = await this.waitBatch(taskId, wait);
+    const items = [];
+    for await (const item of this.listBatchItems(taskId, { limit: 50 }, wait)) items.push(item);
+    return { taskId, report, items };
+  }
+  /**
+   * Save the person's login to a site (a domain or a page URL) from the
+   * Chrome they use, as `octocrawl login import` does, on a server on their
+   * machine. Chrome asks them "Allow remote debugging?": the answer comes
+   * once they click Allow (within `approveTimeoutMs`, default 2 minutes).
+   * The saved login's cookies never leave the server: the answer names the
+   * domain, how many cookies and their hash.
+   */
+  async importLogin(site, opts = {}, request = {}) {
+    return this.post("/v1/logins/import", { ...opts, site }, 200, request, (opts.approveTimeoutMs ?? 12e4) + 3e4);
+  }
+  /** The person's saved logins, without their cookies. */
+  async listLogins(request = {}) {
+    return this.get("/v1/logins", request);
+  }
+  /** Forget a saved login; a W2LError with code `not_found` when none was saved for the site. */
+  async removeLogin(site, request = {}) {
+    const path = `/v1/logins/${encodeURIComponent(site)}`;
+    const res = await this.fetchImpl(`${this.baseUrl}${path}`, { method: "DELETE", headers: this.headers(), signal: request.signal });
+    if (!res.ok) throw await responseError("DELETE", path, res, res.status === 404 ? `no login saved for ${site}` : void 0);
+    return await res.json();
+  }
+  /**
+   * Hands a finished batch's items that a check stopped (a captcha, a
+   * challenge, a login wall) to the person in their own Chrome, on a local
+   * server: each opens in a new tab, they get through it, and W2L reads the
+   * page there. Answers when every item is read or given up, so it waits for
+   * the person: `waitMs` is how long, per page (default 10 minutes). On
+   * Node the SDK waits for the answer as long as that takes (no 300 s limit
+   * on the response headers); `request.signal` ends the wait.
+   */
+  async handOffBatch(id, body = {}, request = {}) {
+    return this.post(`/v1/batches/${encodeURIComponent(id)}/handoff`, body, 200, request, 0);
+  }
+  async cancelBatch(id, request = {}) {
+    return this.post(`/v1/batches/${encodeURIComponent(id)}/cancel`, void 0, 200, request);
+  }
+  /**
+   * Watches a crawl (`kind: 'crawl'`, the default) or a batch (`kind: 'batch'`)
+   * as it runs: `document` events with each page as it is recorded, `snapshot`
+   * events with the report, one `done` with the terminal report, or `error`.
+   * `transport: 'auto'` (default) tries the WebSocket route, then server-sent
+   * events, then polling (`pollIntervalMs`, default 2000, at least 250), each
+   * taking over from the last document seen; `timeoutMs` ends the watch with a
+   * `watcher_timeout` error while the job keeps running. `close()` stops
+   * watching only; cancelCrawl / cancelBatch stay explicit.
+   */
+  watcher(jobId, options = {}) {
+    return new JobWatcher(this.watcherClient(), jobId, options);
+  }
+  /** Starts a crawl and returns its watcher (as `crawl()` then `watcher(taskId, { kind: 'crawl' })`). */
+  async crawlAndWatch(url, opts = {}, watch = {}, request = {}) {
+    const { taskId } = await this.crawl(url, opts, request);
+    return this.watcher(taskId, { ...watch, kind: "crawl" });
+  }
+  /** Starts a batch and returns its watcher (as `batchScrape()` then `watcher(taskId, { kind: 'batch' })`). */
+  async batchScrapeAndWatch(urls, opts = {}, watch = {}, request = {}) {
+    const { taskId } = await this.batchScrape(urls, opts, request);
+    return this.watcher(taskId, { ...watch, kind: "batch" });
+  }
+  /** What a watcher needs of this client: the server, the token and fetch it was given, and the routes it polls, each carrying the bearer header. */
+  watcherClient() {
+    return {
+      baseUrl: this.baseUrl,
+      token: this.token,
+      fetch: this.fetchImpl,
+      headers: (extra) => this.headers(extra),
+      getCrawl: (id, request) => this.getCrawl(id, request),
+      getBatch: (id, request) => this.getBatch(id, request),
+      getCrawlPages: (id, options, request) => this.getCrawlPages(id, options, request),
+      getCrawlErrors: (id, options, request) => this.getCrawlErrors(id, options, request),
+      getBatchItems: (id, options, request) => this.getBatchItems(id, options, request)
+    };
+  }
+  async getCrawl(id, request = {}) {
+    return this.get(`/v1/crawl/${encodeURIComponent(id)}`, request, `crawl not found: ${id}`);
+  }
+  /** The crawls the API process is running, with each one's start URL, status, pages so far and options; empty when nothing runs. */
+  async getActiveCrawls(request = {}) {
+    return this.get("/v1/crawl/active", request);
+  }
+  /** Polls a crawl until it completes, fails or is cancelled. Pages come from listCrawlPages. */
+  async waitCrawl(id, options = {}) {
+    return this.waitFor(id, (request) => this.getCrawl(id, request), options);
+  }
+  /** Starts a crawl, waits for it (as waitCrawl) and lists every page and every error of its latest attempt. */
+  async crawlAndWait(url, opts = {}, wait = {}) {
+    checkWaitOptions(wait);
+    const { taskId } = await this.crawl(url, opts, wait);
+    const report = await this.waitCrawl(taskId, wait);
+    const pages = [];
+    for await (const page of this.listCrawlPages(taskId, { limit: 100 }, wait)) pages.push(page);
+    const errors = [];
+    let cursor;
+    do {
+      const page = await this.getCrawlErrors(taskId, { limit: 100, cursor }, wait);
+      errors.push(...page.items);
+      cursor = page.hasMore ? page.nextCursor ?? void 0 : void 0;
+      if (page.hasMore && cursor === void 0) throw new Error("crawl errors response omitted nextCursor");
+    } while (cursor !== void 0);
+    return { taskId, report, pages, errors };
+  }
+  async getCrawlPages(id, options = {}, request = {}) {
+    return this.getPageList(`/v1/crawl/${encodeURIComponent(id)}/pages`, options, request);
+  }
+  /**
+   * A crawl's pages, page by page, or as many as the PaginationLimits allow;
+   * the generator's return value says where it stopped. The latest attempt's
+   * pages unless `attemptId` names another; a resume with `useCached` records
+   * the pages it reuses in its new attempt, so that attempt normally holds
+   * every page.
+   */
+  listCrawlPages(id, options = {}, request = {}) {
+    return this.paginate((query) => this.getCrawlPages(id, query, request), options, request, CRAWL_PAGE_MAX_LIMIT, "crawl pages");
+  }
+  /** The pages listCrawlPages would yield under the same options, collected, with where the listing stopped. */
+  async collectCrawlPages(id, options = {}, request = {}) {
+    return collect(this.listCrawlPages(id, options, request));
+  }
+  /** A crawl's status and its pages in one answer, every page unless the PaginationLimits stop the listing. */
+  async getCrawlDocuments(id, options = {}, request = {}) {
+    const report = await this.getCrawl(id, request);
+    const { items, nextCursor, stoppedBy } = await this.collectCrawlPages(id, options, request);
+    return { report, pages: items, nextCursor, stoppedBy };
+  }
+  /**
+   * Follow a listing's cursors within its limits. Each page is requested no
+   * larger than the items still wanted, so a stop at `maxResults` leaves a
+   * cursor that continues exactly after the last item returned. A page's
+   * `hasMore` without a cursor is the API breaking its contract and throws.
+   */
+  async *paginate(fetchPage, options, request, maxLimit, what) {
+    checkPaginationLimits(options);
+    const { maxPages, maxResults, maxWaitMs, ...query } = options;
+    const startedAt = Date.now();
+    let cursor;
+    let pagesAfterFirst = 0;
+    let returned = 0;
+    for (; ; ) {
+      const remaining = maxResults === void 0 ? void 0 : maxResults - returned;
+      const limit = remaining === void 0 ? query.limit : Math.min(remaining, query.limit ?? maxLimit);
+      const page = await fetchPage({ ...query, ...limit === void 0 ? {} : { limit }, ...cursor === void 0 ? {} : { cursor } });
+      for (const item of page.items) {
+        if (remaining !== void 0 && returned >= maxResults) break;
+        request.signal?.throwIfAborted();
+        returned++;
+        yield item;
+      }
+      const next = page.hasMore ? page.nextCursor ?? void 0 : void 0;
+      if (page.hasMore && next === void 0) throw new Error(`${what} response omitted nextCursor`);
+      if (next === void 0) return { nextCursor: null, stoppedBy: "end" };
+      if (maxResults !== void 0 && returned >= maxResults) return { nextCursor: next, stoppedBy: "maxResults" };
+      if (maxPages !== void 0 && pagesAfterFirst >= maxPages) return { nextCursor: next, stoppedBy: "maxPages" };
+      if (maxWaitMs !== void 0 && Date.now() - startedAt >= maxWaitMs) return { nextCursor: next, stoppedBy: "maxWait" };
+      cursor = next;
+      pagesAfterFirst++;
+    }
+  }
+  async getCrawlErrors(id, options = {}, request = {}) {
+    return this.getPageList(`/v1/crawl/${encodeURIComponent(id)}/errors`, options, request);
+  }
+  async cancelCrawl(id, request = {}) {
+    return this.post(`/v1/crawl/${encodeURIComponent(id)}/cancel`, void 0, 200, request);
+  }
+  /** Restarts a paused or failed crawl with the options it was started with; follow it with waitCrawl. */
+  async resumeCrawl(id, request = {}) {
+    return this.post(`/v1/crawl/${encodeURIComponent(id)}/resume`, void 0, 202, request);
+  }
+  async createMonitor(input, request = {}) {
+    return this.post("/v1/monitors", input, 201, request);
+  }
+  async previewMonitor(input, request = {}) {
+    return this.post("/v1/monitors/preview", input, 200, request);
+  }
+  async reviseMonitor(id, input, request = {}) {
+    return this.post(`/v1/monitors/${encodeURIComponent(id)}/revisions`, input, 201, request);
+  }
+  async listMonitors(request = {}) {
+    return this.get("/v1/monitors", request);
+  }
+  async getMonitor(id, request = {}) {
+    return this.get(`/v1/monitors/${encodeURIComponent(id)}`, request);
+  }
+  async getMonitorRun(id, runId, request = {}) {
+    return this.get(`/v1/monitors/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}`, request);
+  }
+  /** Durable run: returns after enqueue; client disconnect does not cancel it. */
+  async enqueueMonitorRun(id, input = {}, request = {}) {
+    return this.post(`/v1/monitors/${encodeURIComponent(id)}/runs`, input, 202, request);
+  }
+  /** Waits for capture and assessment; baseline/events are included in the returned view. */
+  async runMonitor(id, input = {}, request = {}) {
+    return this.post(`/v1/monitors/${encodeURIComponent(id)}/run`, input, 200, request);
+  }
+  async pauseMonitor(id, request = {}) {
+    return this.post(`/v1/monitors/${encodeURIComponent(id)}/pause`, void 0, 200, request);
+  }
+  async resumeMonitor(id, request = {}) {
+    return this.post(`/v1/monitors/${encodeURIComponent(id)}/resume`, void 0, 200, request);
+  }
+  async cancelMonitorRun(id, runId, request = {}) {
+    return this.post(`/v1/monitors/${encodeURIComponent(id)}/runs/${encodeURIComponent(runId)}/cancel`, void 0, 200, request);
+  }
+  async createDeliveryDestination(input, request = {}) {
+    return this.post("/v1/delivery/destinations", input, 201, request);
+  }
+  /** The destinations of a Monitor (`monitorId`) or of a crawl or batch (`jobId`); every destination when neither is given. Header names only, never their values. */
+  async listDeliveryDestinations(options = {}, request = {}) {
+    const params = new URLSearchParams();
+    if (options.monitorId !== void 0) params.set("monitorId", options.monitorId);
+    if (options.jobId !== void 0) params.set("jobId", options.jobId);
+    return this.get(`/v1/delivery/destinations${params.size === 0 ? "" : `?${params}`}`, request);
+  }
+  async pauseDeliveryDestination(id, request = {}) {
+    return this.post(`/v1/delivery/destinations/${encodeURIComponent(id)}/pause`, void 0, 200, request);
+  }
+  async resumeDeliveryDestination(id, request = {}) {
+    return this.post(`/v1/delivery/destinations/${encodeURIComponent(id)}/resume`, void 0, 200, request);
+  }
+  /** The deliveries of a Monitor (`monitorId`) or of a crawl or batch (`jobId`, the task id), each with its payload. */
+  async listDeliveries(options = {}, request = {}) {
+    const params = new URLSearchParams();
+    if (options.monitorId !== void 0) params.set("monitorId", options.monitorId);
+    if (options.jobId !== void 0) params.set("jobId", options.jobId);
+    if (options.destinationId !== void 0) params.set("destinationId", options.destinationId);
+    if (options.state !== void 0) params.set("state", options.state);
+    return this.get(`/v1/deliveries${params.size === 0 ? "" : `?${params}`}`, request);
+  }
+  async getDeliveriesPage(options = {}, request = {}) {
+    const params = new URLSearchParams();
+    if (options.monitorId !== void 0) params.set("monitorId", options.monitorId);
+    if (options.jobId !== void 0) params.set("jobId", options.jobId);
+    if (options.destinationId !== void 0) params.set("destinationId", options.destinationId);
+    if (options.state !== void 0) params.set("state", options.state);
+    if (options.cursor !== void 0) params.set("cursor", options.cursor);
+    if (options.limit !== void 0) params.set("limit", String(options.limit));
+    return this.get(`/v1/deliveries/page${params.size ? `?${params}` : ""}`, request);
+  }
+  async getDelivery(id, request = {}) {
+    return this.get(`/v1/deliveries/${encodeURIComponent(id)}`, request);
+  }
+  async retryDelivery(id, request = {}) {
+    return this.post(`/v1/deliveries/${encodeURIComponent(id)}/retry`, void 0, 200, request);
+  }
+  async waitFor(id, poll, options) {
+    checkWaitOptions(options);
+    const deadline = options.timeoutMs === void 0 ? void 0 : Date.now() + options.timeoutMs;
+    const expiry = new AbortController();
+    let timer;
+    const arm = () => {
+      timer = setTimeout(() => {
+        if (Date.now() >= deadline) expiry.abort(new DOMException("wait timed out", "TimeoutError"));
+        else arm();
+      }, Math.min(Math.max(0, deadline - Date.now()), 2147483647));
+    };
+    if (deadline !== void 0) arm();
+    const signal = options.signal === void 0 ? expiry.signal : AbortSignal.any([options.signal, expiry.signal]);
+    let last = null;
+    let failure = void 0;
+    let failures = 0;
+    const timedOut = () => new WaitTimeoutError(id, last, options.timeoutMs, failure === void 0 ? void 0 : { cause: failure });
+    try {
+      for (; ; ) {
+        options.signal?.throwIfAborted();
+        let pause = options.pollIntervalMs ?? 500;
+        try {
+          const report = await poll({ signal });
+          if (FINISHED2.includes(report.status)) return report;
+          last = report;
+          failure = void 0;
+          failures = 0;
+        } catch (error) {
+          options.signal?.throwIfAborted();
+          if (expiry.signal.aborted) throw timedOut();
+          const delay = failures < (options.maxRetries ?? 5) ? retryDelayMs(error, failures + 1) : null;
+          if (delay === null) throw error;
+          failure = error;
+          failures++;
+          pause = delay;
+        }
+        if (deadline !== void 0 && Date.now() >= deadline) throw timedOut();
+        try {
+          await sleep2(Math.min(pause, deadline === void 0 ? Infinity : deadline - Date.now()), signal);
+        } catch (error) {
+          options.signal?.throwIfAborted();
+          if (expiry.signal.aborted) throw timedOut();
+          throw error;
+        }
+      }
+    } finally {
+      if (timer !== void 0) clearTimeout(timer);
+    }
+  }
+  headers(extra = {}) {
+    return this.token === void 0 || this.token.length === 0 ? extra : { ...extra, authorization: `Bearer ${this.token}` };
+  }
+  /** `answerWithinMs`: how long the platform's fetch waits for the response headers, on Node instead of undici's 300 s. */
+  async post(path, body, ok = 200, request = {}, answerWithinMs) {
+    const dispatcher = answerWithinMs === void 0 || !this.platformFetch ? void 0 : headersWait(answerWithinMs);
+    const init = {
+      method: "POST",
+      signal: request.signal,
+      headers: this.headers({ "content-type": "application/json" }),
+      ...body === void 0 ? {} : { body: JSON.stringify(body) },
+      ...dispatcher === void 0 ? {} : { dispatcher }
+    };
+    const res = await this.fetchImpl(`${this.baseUrl}${path}`, init);
+    if (res.status !== ok) throw await responseError("POST", path, res);
+    return await res.json();
+  }
+  async getPageList(path, options, request) {
+    const params = new URLSearchParams();
+    if (options.cursor !== void 0) params.set("cursor", options.cursor);
+    if (options.limit !== void 0) params.set("limit", String(options.limit));
+    if (options.attemptId !== void 0) params.set("attemptId", options.attemptId);
+    if (options.debug !== void 0) params.set("debug", String(options.debug));
+    if (options.includeDuplicates !== void 0) params.set("includeDuplicates", String(options.includeDuplicates));
+    const suffix = params.size === 0 ? "" : `?${params.toString()}`;
+    return this.get(`${path}${suffix}`, request, `crawl not found: ${path}`);
+  }
+  async get(path, request, notFound) {
+    const res = await this.fetchImpl(`${this.baseUrl}${path}`, { headers: this.headers(), signal: request.signal });
+    if (res.status === 404 && notFound !== void 0) throw await responseError("GET", path, res, notFound);
+    if (!res.ok) throw await responseError("GET", path, res);
+    return await res.json();
+  }
+};
+async function collect(listing) {
+  const items = [];
+  for (; ; ) {
+    const next = await listing.next();
+    if (next.done) return { items, nextCursor: next.value.nextCursor, hasMore: next.value.nextCursor !== null, stoppedBy: next.value.stoppedBy };
+    items.push(next.value);
+  }
+}
+function originOf(opts, request) {
+  return opts.origin ?? request.origin ?? SDK_ORIGIN;
+}
+function sleep2(ms, signal) {
+  return new Promise((resolve, reject) => {
+    if (signal.aborted) {
+      reject(signal.reason);
+      return;
+    }
+    const timer = setTimeout(() => {
+      signal.removeEventListener("abort", abort);
+      resolve();
+    }, Math.max(0, ms));
+    const abort = () => {
+      clearTimeout(timer);
+      reject(signal.reason);
+    };
+    signal.addEventListener("abort", abort, { once: true });
+  });
+}
+export {
+  DEFAULT_WATCH_POLL_INTERVAL_MS,
+  JobWatcher,
+  MAP_ANSWER_MARGIN_MS,
+  MIN_WATCH_POLL_INTERVAL_MS,
+  SDK_ORIGIN,
+  SDK_VERSION,
+  W2L,
+  W2LError,
+  WaitTimeoutError,
+  chunkUrls,
+  pageCursor,
+  parseSseBlock
+};