@crawlora-org/sdk 1.3.0-sdk.1 → 1.5.0-sdk.2

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## v1.5.0-sdk.1
+
+- Added `beforeRequest`/`afterResponse` middleware hooks (mutate the outgoing
+  request context, transform the parsed response).
+- Added opt-in `idempotencyKeys` (stable `Idempotency-Key` on POST/PATCH, reused
+  across retries) and per-request `retries`/`isRetryable` overrides.
+- Added client-side `rateLimit` (requests/sec) and `maxConcurrency` throttling.
+- Errors now carry `requestId`.
+
+## v1.4.0-sdk.1
+
+- Configurable retries: `maxRetryDelay`, `retryStatuses`, and an `isRetryable`
+  predicate; added an `onRetry` hook, opt-in `requestId` (x-request-id, also on
+  `error.requestId`), and a `logger` event sink.
+- Pagination: cursor/token mode (`cursorParam` + `nextCursor`) and a
+  `paginateItems` per-item async iterator; operation metadata now exposes
+  `paginatable` and `cursorParams`.
+- Streaming: `responseType: "stream"` returns the raw `Response` on success.
+- Config: `CRAWLORA_API_KEY` / `CRAWLORA_BASE_URL` environment fallback.
+
 ## v1.3.0-sdk.1
 
 - Added `CrawloraClientError`, `CrawloraServerError`, and `CrawloraNetworkError`

package/docs/recipes.md

@@ -74,6 +74,77 @@ try {
 }
 ```
 
+## Custom Retries And Observability
+
+```js
+const crawlora = new CrawloraClient({
+  retries: 3,
+  maxRetryDelay: 10000,
+  retryStatuses: [429, 503],                       // or:
+  isRetryable: (status, error) => status >= 500,
+  onRetry: (attempt, error, delay) => console.warn(`retry ${attempt} after ${delay}ms`, error.status),
+  requestId: true,                                  // sets x-request-id; available as error.requestId
+  logger: (event) => console.debug(event)
+});
+```
+
+Branch on `CrawloraClientError` (4xx), `CrawloraServerError` (5xx), and
+`CrawloraNetworkError` (transport).
+
+## Pagination
+
+```js
+// page/offset (auto-detected)
+for await (const page of crawlora.paginate("ebay-seller-feedback", { seller: "acme" })) { /* ... */ }
+
+// per-item iteration
+for await (const item of crawlora.paginateItems("ebay-seller-feedback", { seller: "acme" })) { /* ... */ }
+
+// cursor/token pagination
+for await (const page of crawlora.paginate("producthunt-leaderboard", {}, {
+  cursorParam: "cursor",
+  nextCursor: (p) => p.next_cursor
+})) { /* ... */ }
+```
+
+## Streaming Responses
+
+```js
+const res = await crawlora.request("bing-search", { q: "coffee" }, { responseType: "stream" });
+for await (const chunk of res.body) { /* ... */ }
+```
+
+## Environment Variables
+
+`CRAWLORA_API_KEY` and `CRAWLORA_BASE_URL` are used when not set explicitly
+(precedence: option > env > default).
+
+## Middleware
+
+```js
+const crawlora = new CrawloraClient({
+  beforeRequest: (ctx) => { ctx.headers["x-signature"] = sign(ctx); },
+  afterResponse: (operationId, status, headers, body) => body // return a value to transform
+});
+```
+
+## Idempotency And Per-Request Retries
+
+```js
+const crawlora = new CrawloraClient({ idempotencyKeys: true }); // stable key on POST/PATCH retries
+
+await crawlora.request("bing-search", { q: "coffee" }, {
+  retries: 5,
+  isRetryable: (status) => status >= 500
+});
+```
+
+## Rate Limiting
+
+```js
+const crawlora = new CrawloraClient({ rateLimit: 10, maxConcurrency: 4 });
+```
+
 ## Optional Live Smoke Tests
 
 ```sh

package/examples/paginate.mjs

@@ -0,0 +1,22 @@
+import { CrawloraClient } from "../src/index.js";
+
+const client = newClient();
+if (client) {
+  const seller = process.env.CRAWLORA_EBAY_SELLER || "garlandcomputer";
+  let count = 0;
+  for await (const item of client.paginateItems("ebay-seller-feedback", { seller }, { maxPages: 3 })) {
+    count += 1;
+  }
+  console.log(`collected ${count} feedback items across up to 3 pages`);
+}
+
+function newClient() {
+  if (!process.env.CRAWLORA_API_KEY) {
+    console.error("set CRAWLORA_API_KEY to run this live example");
+    return null;
+  }
+  return new CrawloraClient({
+    apiKey: process.env.CRAWLORA_API_KEY,
+    baseUrl: process.env.CRAWLORA_BASE_URL
+  });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crawlora-org/sdk",
-  "version": "1.3.0-sdk.1",
+  "version": "1.5.0-sdk.2",
   "description": "JavaScript and TypeScript SDK for the public Crawlora API.",
   "keywords": [
     "crawlora",

package/src/client.js

@@ -1,11 +1,11 @@
 import { operations, groups } from "./operations.js";
 
 const DEFAULT_BASE_URL = "https://api.crawlora.net/api/v1";
-export const VERSION = "1.3.0-sdk.1";
+export const VERSION = "1.5.0-sdk.2";
 const DEFAULT_USER_AGENT = `crawlora-js-sdk/${VERSION}`;
 
 export class CrawloraError extends Error {
-  constructor(message, { status = 0, code, body, headers = {}, response, cause, retryable } = {}) {
+  constructor(message, { status = 0, code, body, headers = {}, response, cause, retryable, requestId } = {}) {
     super(message);
     this.name = "CrawloraError";
     this.status = status;
@@ -15,6 +15,7 @@ export class CrawloraError extends Error {
     this.response = response;
     this.cause = cause;
     this.retryable = retryable;
+    this.requestId = requestId;
   }
 }
 
@@ -47,14 +48,37 @@ function apiErrorClass(status) {
   return status >= 500 ? CrawloraServerError : CrawloraClientError;
 }
 
+/**
+ * Client for the Crawlora API.
+ *
+ * Call operations via grouped helpers (`client.bing.search({ q })`) or
+ * dynamically (`client.request("bing-search", { q })`). Supports configurable
+ * retries, an `onRetry` hook, opt-in `requestId` and `idempotencyKeys`,
+ * `beforeRequest`/`afterResponse` middleware, client-side `rateLimit` /
+ * `maxConcurrency`, pagination (`paginate`/`paginateItems`), and
+ * `responseType: "stream"`.
+ */
 export class CrawloraClient {
   constructor(options = {}) {
-    this.apiKey = options.apiKey || "";
+    // Precedence: explicit option > environment variable > default.
+    this.apiKey = options.apiKey || envVar("CRAWLORA_API_KEY") || "";
     this.jwtToken = options.jwtToken || "";
-    this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, "");
+    this.baseUrl = (options.baseUrl || envVar("CRAWLORA_BASE_URL") || DEFAULT_BASE_URL).replace(/\/+$/, "");
     this.timeout = options.timeout ?? 30000;
     this.retries = normalizeNonNegativeInteger(options.retries ?? 0);
     this.retryDelay = normalizeNonNegativeNumber(options.retryDelay ?? 250);
+    this.maxRetryDelay = normalizeNonNegativeNumber(options.maxRetryDelay ?? 30000);
+    this.retryStatuses = options.retryStatuses ? new Set(options.retryStatuses) : null;
+    this.isRetryable = typeof options.isRetryable === "function" ? options.isRetryable : null;
+    this.onRetry = typeof options.onRetry === "function" ? options.onRetry : null;
+    this.requestId = options.requestId === true;
+    this.idempotencyKeys = options.idempotencyKeys === true;
+    this.logger = typeof options.logger === "function" ? options.logger : null;
+    this.beforeRequest = asHookList(options.beforeRequest);
+    this.afterResponse = asHookList(options.afterResponse);
+    this.limiter = (options.rateLimit > 0 || options.maxConcurrency > 0)
+      ? new RateLimiter(options.rateLimit || 0, options.maxConcurrency || 0)
+      : null;
     this.headers = { ...(options.headers || {}) };
     this.userAgent = options.userAgent === false ? "" : (options.userAgent || DEFAULT_USER_AGENT);
     this.fetch = options.fetch || globalThis.fetch;
@@ -84,22 +108,47 @@ export class CrawloraClient {
     params = params ?? {};
     options = options ?? {};
     const responseType = validateResponseType(options.responseType || "auto");
+    this.#log({ event: "request", operation: operationId });
+    const maxRetries = options.retries ?? this.retries;
+    const isRetryable = typeof options.isRetryable === "function" ? options.isRetryable : null;
+    const idempotencyKey = (this.idempotencyKeys && (operation.method === "POST" || operation.method === "PATCH"))
+      ? generateId() : undefined;
 
     let attempt = 0;
     for (;;) {
       try {
-        return await this.#send(operation, params, options, responseType);
+        const run = () => this.#send(operation, params, options, responseType, idempotencyKey);
+        return await (this.limiter ? this.limiter.run(run, options.signal) : run());
       } catch (error) {
-        if (!(error instanceof CrawloraError) || error.retryable === false || attempt >= this.retries || !shouldRetry(error.status)) {
+        const retryable = isRetryable ? !!isRetryable(error.status, error) : this.#isRetryable(error.status, error);
+        if (!(error instanceof CrawloraError) || error.retryable === false || attempt >= maxRetries || !retryable) {
           throw error;
         }
         attempt++;
-        await sleep(retryDelayForError(error, this.retryDelay, attempt), options.signal);
+        const delay = this.#retryDelayFor(error, attempt);
+        this.#log({ event: "retry", operation: operationId, attempt, status: error.status, delay });
+        if (this.onRetry) this.onRetry(attempt, error, delay);
+        await sleep(delay, options.signal);
       }
     }
   }
 
-  async #send(operation, params, options, responseType) {
+  #isRetryable(status, error) {
+    if (this.isRetryable) return !!this.isRetryable(status, error);
+    if (this.retryStatuses) return status === 0 || this.retryStatuses.has(status);
+    return shouldRetry(status);
+  }
+
+  #retryDelayFor(error, attempt) {
+    const retryAfter = parseRetryAfter(error.headers, this.maxRetryDelay);
+    return retryAfter ?? retryDelay(this.retryDelay, attempt);
+  }
+
+  #log(event) {
+    if (this.logger) this.logger(event);
+  }
+
+  async #send(operation, params, options, responseType, idempotencyKey) {
     const { url, body, bodyHeaders } = buildRequest(operation, this.baseUrl, params);
     const headers = mergeHeaders(
       this.headers,
@@ -108,6 +157,19 @@ export class CrawloraClient {
       bodyHeaders,
       options.headers
     );
+    const requestId = this.requestId ? ensureRequestId(headers) : (headerValue(headers, "x-request-id") || undefined);
+    if (idempotencyKey && !headerValue(headers, "idempotency-key")) {
+      headers["Idempotency-Key"] = idempotencyKey;
+    }
+
+    let requestUrl = url;
+    let requestHeaders = headers;
+    if (this.beforeRequest.length) {
+      const ctx = { operationId: operation.id, method: operation.method, url: requestUrl, headers: requestHeaders };
+      for (const hook of this.beforeRequest) await hook(ctx);
+      requestUrl = ctx.url;
+      requestHeaders = ctx.headers;
+    }
 
     const controller = new AbortController();
     const timeoutMs = options.timeout ?? this.timeout;
@@ -119,26 +181,30 @@ export class CrawloraClient {
     const signal = composeSignal(controller, options.signal);
     let response;
     try {
-      response = await this.fetch(url, {
+      response = await this.fetch(requestUrl, {
         method: operation.method,
-        headers,
+        headers: requestHeaders,
         body,
         signal
       });
     } catch (error) {
       if (options.signal?.aborted) {
-        throw new CrawloraNetworkError("Crawlora request aborted", { cause: error, retryable: false });
+        throw new CrawloraNetworkError("Crawlora request aborted", { cause: error, retryable: false, requestId });
       }
       if (timedOut) {
-        throw new CrawloraNetworkError("Crawlora request timed out", { cause: error, retryable: false });
+        throw new CrawloraNetworkError("Crawlora request timed out", { cause: error, retryable: false, requestId });
       }
-      throw new CrawloraNetworkError("Crawlora transport error", { cause: error });
+      throw new CrawloraNetworkError("Crawlora transport error", { cause: error, requestId });
     } finally {
       if (timeout) clearTimeout(timeout);
     }
 
     const headersObject = responseHeaders(response);
-    const parsed = await parseResponse(response, responseType, headersObject);
+    // Streaming success returns the raw Response; the caller reads response.body.
+    if (responseType === "stream" && response.ok) {
+      return response;
+    }
+    let parsed = await parseResponse(response, responseType === "stream" ? "auto" : responseType, headersObject, requestId);
     if (!response.ok) {
       const code = parsed && typeof parsed === "object" ? parsed.code : undefined;
       const message = parsed && typeof parsed === "object" && parsed.msg ? parsed.msg : response.statusText;
@@ -148,26 +214,54 @@ export class CrawloraClient {
         code,
         body: parsed,
         headers: headersObject,
-        response
+        response,
+        requestId
       });
     }
+    if (this.afterResponse.length) {
+      for (const hook of this.afterResponse) {
+        const result = await hook(operation.id, response.status, headersObject, parsed);
+        if (result !== undefined) parsed = result;
+      }
+    }
     return parsed;
   }
 
-  // Async iterator over pages of a paginated operation. Advances the numeric
-  // page/offset query parameter and stops when a page returns no data.
+  // Async iterator over pages of a paginated operation. Numeric mode advances
+  // the page/offset query parameter and stops on an empty page; cursor mode
+  // (cursorParam + nextCursor) sends the cursor and stops when nextCursor is falsy.
   //   for await (const page of client.paginate("ebay-seller-feedback", { seller })) { ... }
   async *paginate(operationId, params = {}, options = {}) {
     const operation = operations[operationId];
     if (!operation) {
       throw new TypeError(`Unknown Crawlora operation: ${operationId}`);
     }
+    const maxPages = options.maxPages ?? Infinity;
+
+    if (options.cursorParam || options.nextCursor) {
+      if (!(options.cursorParam && options.nextCursor)) {
+        throw new TypeError("cursor pagination requires both cursorParam and nextCursor");
+      }
+      if (!operation.queryParams.some((parameter) => parameter.name === options.cursorParam)) {
+        throw new TypeError(`cursorParam ${options.cursorParam} is not a query parameter of ${operationId}`);
+      }
+      let cursor = options.start;
+      for (let i = 0; i < maxPages; i++) {
+        const pageParams = { ...params };
+        if (cursor !== undefined && cursor !== null) pageParams[options.cursorParam] = cursor;
+        const response = await this.request(operationId, pageParams, options);
+        yield response;
+        cursor = options.nextCursor(response);
+        if (!cursor) break;
+      }
+      return;
+    }
+
     const pageParam = options.pageParam || detectPageParam(operation);
     if (!pageParam) {
       throw new TypeError(`Operation ${operationId} has no page or offset query parameter to paginate`);
     }
     const step = options.step ?? 1;
-    const maxPages = options.maxPages ?? Infinity;
     let pageValue = options.start ?? (pageParam === "offset" ? 0 : 1);
     for (let i = 0; i < maxPages; i++) {
       const response = await this.request(operationId, { ...params, [pageParam]: pageValue }, options);
@@ -176,10 +270,25 @@ export class CrawloraClient {
       pageValue += step;
     }
   }
+
+  // Async iterator over individual items across pages. `items` extracts the list
+  // from a page (default: the Crawlora `data` array).
+  async *paginateItems(operationId, params = {}, options = {}) {
+    const extract = options.items || defaultItems;
+    for await (const page of this.paginate(operationId, params, options)) {
+      for (const item of extract(page)) yield item;
+    }
+  }
 }
 
 const PAGE_PARAM_NAMES = ["page", "offset"];
 
+function defaultItems(response) {
+  if (response && typeof response === "object" && Array.isArray(response.data)) return response.data;
+  if (Array.isArray(response)) return response;
+  return [];
+}
+
 function detectPageParam(operation) {
   for (const name of PAGE_PARAM_NAMES) {
     if (operation.queryParams.some((parameter) => parameter.name === name)) return name;
@@ -316,13 +425,13 @@ function composeSignal(controller, signal) {
 }
 
 function validateResponseType(responseType) {
-  if (responseType === "auto" || responseType === "json" || responseType === "text") {
+  if (responseType === "auto" || responseType === "json" || responseType === "text" || responseType === "stream") {
     return responseType;
   }
-  throw new TypeError("Invalid responseType: expected one of auto, json, text");
+  throw new TypeError("Invalid responseType: expected one of auto, json, text, stream");
 }
 
-async function parseResponse(response, responseType, headers) {
+async function parseResponse(response, responseType, headers, requestId) {
   if (responseType === "text") return response.text();
   const contentType = response.headers.get("content-type") || "";
   if (responseType === "json" || contentType.toLowerCase().includes("application/json")) {
@@ -335,7 +444,8 @@ async function parseResponse(response, responseType, headers) {
         body: text,
         headers,
         response,
-        cause: error
+        cause: error,
+        requestId
       });
     }
   }
@@ -353,25 +463,86 @@ function retryDelay(baseDelay, attempt) {
   return delay + jitter;
 }
 
-function retryDelayForError(error, baseDelay, attempt) {
-  return parseRetryAfter(error.headers) ?? retryDelay(baseDelay, attempt);
-}
-
-function parseRetryAfter(headers) {
+function parseRetryAfter(headers, cap = 30000) {
   const value = headerValue(headers, "retry-after");
   if (!value) return undefined;
   const seconds = Number(value);
   if (Number.isFinite(seconds) && seconds > 0) {
-    return Math.min(seconds * 1000, 30000);
+    return Math.min(seconds * 1000, cap);
   }
   const date = Date.parse(value);
   if (Number.isFinite(date)) {
     const delay = date - Date.now();
-    if (delay > 0) return Math.min(delay, 30000);
+    if (delay > 0) return Math.min(delay, cap);
   }
   return undefined;
 }
 
+function asHookList(value) {
+  if (!value) return [];
+  return typeof value === "function" ? [value] : Array.from(value);
+}
+
+// Optional client-side throttle: caps concurrency and spaces requests to a
+// maximum rate (requests per second).
+class RateLimiter {
+  constructor(rps, concurrency) {
+    this.interval = rps > 0 ? 1000 / rps : 0;
+    this.limit = concurrency > 0 ? concurrency : Infinity;
+    this.active = 0;
+    this.waiters = [];
+    this.nextAllowed = 0;
+  }
+
+  async run(fn, signal) {
+    await this.#acquire();
+    try {
+      await this.#rateWait(signal);
+      return await fn();
+    } finally {
+      const next = this.waiters.shift();
+      if (next) {
+        next(); // transfer the slot to the next waiter (active unchanged)
+      } else {
+        this.active--;
+      }
+    }
+  }
+
+  #acquire() {
+    if (this.active < this.limit) {
+      this.active++;
+      return Promise.resolve();
+    }
+    // Wait for a slot to be transferred to us (active stays at the limit).
+    return new Promise((resolve) => this.waiters.push(resolve));
+  }
+
+  async #rateWait(signal) {
+    if (!this.interval) return;
+    const now = Date.now();
+    const wait = Math.max(0, this.nextAllowed - now);
+    this.nextAllowed = Math.max(now, this.nextAllowed) + this.interval;
+    if (wait > 0) await sleep(wait, signal);
+  }
+}
+
+function envVar(name) {
+  return (typeof process !== "undefined" && process.env && process.env[name]) || undefined;
+}
+
+function generateId() {
+  return (typeof crypto !== "undefined" && crypto.randomUUID) ? crypto.randomUUID() : `id-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+}
+
+function ensureRequestId(headers) {
+  const existing = headerValue(headers, "x-request-id");
+  if (existing) return existing;
+  const id = generateId();
+  headers["x-request-id"] = id;
+  return id;
+}
+
 function headerValue(headers, name) {
   for (const [key, value] of Object.entries(headers || {})) {
     if (key.toLowerCase() === name.toLowerCase()) return value;

package/src/index.d.ts

@@ -8,6 +8,22 @@ import type {
 
 export type CrawloraParams = Record<string, unknown>;
 
+export type CrawloraLogEvent = { event: string; [key: string]: unknown };
+
+export interface CrawloraRequestContext {
+  operationId: string;
+  method: string;
+  url: string;
+  headers: Record<string, string>;
+}
+export type CrawloraBeforeRequest = (ctx: CrawloraRequestContext) => void | Promise<void>;
+export type CrawloraAfterResponse = (
+  operationId: string,
+  status: number,
+  headers: Record<string, string>,
+  body: unknown
+) => unknown;
+
 export interface CrawloraClientOptions {
   apiKey?: string;
   jwtToken?: string;
@@ -15,6 +31,28 @@ export interface CrawloraClientOptions {
   timeout?: number;
   retries?: number;
   retryDelay?: number;
+  /** Cap on backoff and Retry-After delays in ms (default 30000). */
+  maxRetryDelay?: number;
+  /** Override the retryable HTTP status set (network status 0 stays retryable). */
+  retryStatuses?: Iterable<number>;
+  /** Full predicate; supersedes retryStatuses when set. */
+  isRetryable?: (status: number, error: CrawloraError) => boolean;
+  /** Called before each retry sleep with (attempt, error, delayMs). */
+  onRetry?: (attempt: number, error: CrawloraError, delay: number) => void;
+  /** Generate an x-request-id header when absent. */
+  requestId?: boolean;
+  /** Attach a stable Idempotency-Key on POST/PATCH, reused across retries. */
+  idempotencyKeys?: boolean;
+  /** Cap outgoing requests to at most this many per second. */
+  rateLimit?: number;
+  /** Cap the number of in-flight requests. */
+  maxConcurrency?: number;
+  /** Structured event sink (request/retry); never logs on its own. */
+  logger?: (event: CrawloraLogEvent) => void;
+  /** Hook(s) run before each request; mutate ctx.headers / ctx.url. */
+  beforeRequest?: CrawloraBeforeRequest | Iterable<CrawloraBeforeRequest>;
+  /** Hook(s) run on the parsed success body; return a value to replace it. */
+  afterResponse?: CrawloraAfterResponse | Iterable<CrawloraAfterResponse>;
   headers?: Record<string, string>;
   userAgent?: string | false;
   fetch?: typeof globalThis.fetch;
@@ -22,9 +60,13 @@ export interface CrawloraClientOptions {
 
 export interface CrawloraRequestOptions {
   headers?: Record<string, string>;
-  responseType?: "auto" | "json" | "text";
+  responseType?: "auto" | "json" | "text" | "stream";
   timeout?: number;
   signal?: AbortSignal;
+  /** Per-request retry count, overriding the client default. */
+  retries?: number;
+  /** Per-request retry predicate, overriding the client default. */
+  isRetryable?: (status: number, error: CrawloraError) => boolean;
 }
 
 export type OperationMethod = <T = unknown>(
@@ -44,6 +86,8 @@ export interface OperationDefinition {
   consumes: string[];
   produces: string[];
   security: string[];
+  paginatable?: boolean;
+  cursorParams?: string[];
 }
 
 export class CrawloraError extends Error {
@@ -54,6 +98,7 @@ export class CrawloraError extends Error {
   response?: Response;
   cause?: unknown;
   retryable?: boolean;
+  requestId?: string;
 }
 
 // Thrown for 4xx API responses (request rejected by the API).
@@ -64,16 +109,25 @@ export class CrawloraServerError extends CrawloraError {}
 export class CrawloraNetworkError extends CrawloraError {}
 
 export interface CrawloraPaginateOptions extends CrawloraRequestOptions {
-  // Query parameter to advance. Auto-detected as "page" or "offset" when omitted.
+  // Numeric query parameter to advance. Auto-detected as "page" or "offset".
   pageParam?: string;
-  // First value for the page parameter. Defaults to 1 for "page", 0 for "offset".
-  start?: number;
+  // Cursor/token query parameter (cursor mode). Requires nextCursor.
+  cursorParam?: string;
+  // Extracts the next cursor from a page; iteration stops when it returns falsy.
+  nextCursor?: (page: unknown) => unknown;
+  // First page value (numeric) or initial cursor value.
+  start?: unknown;
   // Amount added to the page parameter after each page. Defaults to 1.
   step?: number;
   // Maximum number of pages to fetch. Defaults to unbounded.
   maxPages?: number;
 }
 
+export interface CrawloraPaginateItemsOptions extends CrawloraPaginateOptions {
+  // Extracts the item list from a page (default: the `data` array).
+  items?: (page: unknown) => Iterable<unknown>;
+}
+
 export class CrawloraClient {
   constructor(options?: CrawloraClientOptions);
   request<I extends OperationId>(
@@ -89,6 +143,11 @@ export class CrawloraClient {
     params?: OperationParamsMap[I],
     options?: CrawloraPaginateOptions
   ): AsyncGenerator<OperationResponseMap[I], void, unknown>;
+  paginateItems<I extends OperationId>(
+    operationId: I,
+    params?: OperationParamsMap[I],
+    options?: CrawloraPaginateItemsOptions
+  ): AsyncGenerator<unknown, void, unknown>;
   [group: string]: unknown;
 }
 

package/src/operations.js

@@ -51,6 +51,7 @@ export const operations = {
     "formParams": [],
     "id": "airbnb-room-reviews",
     "method": "GET",
+    "paginatable": true,
     "path": "/airbnb/room/{id}/reviews",
     "pathParams": [
       "id"
@@ -78,6 +79,7 @@ export const operations = {
     "formParams": [],
     "id": "airbnb-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/airbnb/search",
     "pathParams": [],
     "produces": [
@@ -192,6 +194,7 @@ export const operations = {
     "formParams": [],
     "id": "amazon-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/amazon/search",
     "pathParams": [],
     "produces": [
@@ -289,6 +292,7 @@ export const operations = {
     "formParams": [],
     "id": "apple-podcasts-episodes-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/apple-podcasts/episodes/search",
     "pathParams": [],
     "produces": [
@@ -335,6 +339,7 @@ export const operations = {
     "formParams": [],
     "id": "apple-podcasts-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/apple-podcasts/search",
     "pathParams": [],
     "produces": [
@@ -649,6 +654,7 @@ export const operations = {
     "formParams": [],
     "id": "appstore-reviews",
     "method": "GET",
+    "paginatable": true,
     "path": "/appstore/reviews",
     "pathParams": [],
     "produces": [
@@ -703,6 +709,7 @@ export const operations = {
     "formParams": [],
     "id": "appstore-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/appstore/search",
     "pathParams": [],
     "produces": [
@@ -1070,6 +1077,7 @@ export const operations = {
     "formParams": [],
     "id": "bing-images",
     "method": "GET",
+    "paginatable": true,
     "path": "/bing/images",
     "pathParams": [],
     "produces": [
@@ -1116,6 +1124,7 @@ export const operations = {
     "formParams": [],
     "id": "bing-news",
     "method": "GET",
+    "paginatable": true,
     "path": "/bing/news",
     "pathParams": [],
     "produces": [
@@ -1162,6 +1171,7 @@ export const operations = {
     "formParams": [],
     "id": "bing-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/bing/search",
     "pathParams": [],
     "produces": [
@@ -1249,6 +1259,7 @@ export const operations = {
     "formParams": [],
     "id": "bing-videos",
     "method": "GET",
+    "paginatable": true,
     "path": "/bing/videos",
     "pathParams": [],
     "produces": [
@@ -1295,6 +1306,7 @@ export const operations = {
     "formParams": [],
     "id": "brave-images",
     "method": "GET",
+    "paginatable": true,
     "path": "/brave/images",
     "pathParams": [],
     "produces": [
@@ -1397,6 +1409,7 @@ export const operations = {
     "formParams": [],
     "id": "brave-news",
     "method": "GET",
+    "paginatable": true,
     "path": "/brave/news",
     "pathParams": [],
     "produces": [
@@ -1522,6 +1535,7 @@ export const operations = {
     "formParams": [],
     "id": "brave-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/brave/search",
     "pathParams": [],
     "produces": [
@@ -1739,6 +1753,7 @@ export const operations = {
     "formParams": [],
     "id": "brave-videos",
     "method": "GET",
+    "paginatable": true,
     "path": "/brave/videos",
     "pathParams": [],
     "produces": [
@@ -1959,6 +1974,7 @@ export const operations = {
     "formParams": [],
     "id": "coingecko-category-coins",
     "method": "GET",
+    "paginatable": true,
     "path": "/coingecko/category/{slug}/coins",
     "pathParams": [
       "slug"
@@ -2548,6 +2564,7 @@ export const operations = {
     "formParams": [],
     "id": "coingecko-exchanges",
     "method": "GET",
+    "paginatable": true,
     "path": "/coingecko/exchanges",
     "pathParams": [],
     "produces": [
@@ -2863,6 +2880,7 @@ export const operations = {
     "formParams": [],
     "id": "coingecko-markets",
     "method": "GET",
+    "paginatable": true,
     "path": "/coingecko/markets",
     "pathParams": [],
     "produces": [
@@ -2963,6 +2981,7 @@ export const operations = {
     "formParams": [],
     "id": "coingecko-new-coins",
     "method": "GET",
+    "paginatable": true,
     "path": "/coingecko/new-coins",
     "pathParams": [],
     "produces": [
@@ -3088,6 +3107,7 @@ export const operations = {
     "formParams": [],
     "id": "coingecko-nft-category",
     "method": "GET",
+    "paginatable": true,
     "path": "/coingecko/nft/category/{slug}",
     "pathParams": [
       "slug"
@@ -3190,6 +3210,7 @@ export const operations = {
     "formParams": [],
     "id": "coingecko-nfts",
     "method": "GET",
+    "paginatable": true,
     "path": "/coingecko/nfts",
     "pathParams": [],
     "produces": [
@@ -3682,6 +3703,7 @@ export const operations = {
     "formParams": [],
     "id": "datasets-google-map-businesses-nearby",
     "method": "GET",
+    "paginatable": true,
     "path": "/datasets/google-map-businesses/nearby",
     "pathParams": [],
     "produces": [
@@ -3745,6 +3767,7 @@ export const operations = {
     "formParams": [],
     "id": "datasets-google-map-businesses-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/datasets/google-map-businesses/search",
     "pathParams": [],
     "produces": [
@@ -3951,6 +3974,7 @@ export const operations = {
     "formParams": [],
     "id": "ebay-seller-feedback",
     "method": "GET",
+    "paginatable": true,
     "path": "/ebay/seller/{seller}/feedback",
     "pathParams": [
       "seller"
@@ -3988,6 +4012,7 @@ export const operations = {
     "formParams": [],
     "id": "ebay-seller-shop",
     "method": "GET",
+    "paginatable": true,
     "path": "/ebay/seller/{seller}/shop",
     "pathParams": [
       "seller"
@@ -4340,6 +4365,7 @@ export const operations = {
     "formParams": [],
     "id": "google-finance-markets-category-news",
     "method": "GET",
+    "paginatable": true,
     "path": "/google/finance/markets/categories/{category}/news",
     "pathParams": [
       "category"
@@ -4367,6 +4393,7 @@ export const operations = {
     "formParams": [],
     "id": "google-finance-markets-category-stocks",
     "method": "GET",
+    "paginatable": true,
     "path": "/google/finance/markets/categories/{category}/stocks",
     "pathParams": [
       "category"
@@ -4470,6 +4497,7 @@ export const operations = {
     "formParams": [],
     "id": "google-finance-markets-movers",
     "method": "GET",
+    "paginatable": true,
     "path": "/google/finance/markets/movers",
     "pathParams": [],
     "produces": [
@@ -4505,6 +4533,7 @@ export const operations = {
     "formParams": [],
     "id": "google-finance-markets-top",
     "method": "GET",
+    "paginatable": true,
     "path": "/google/finance/markets/top",
     "pathParams": [],
     "produces": [
@@ -6773,9 +6802,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "producthunt-alternatives",
     "method": "GET",
+    "paginatable": true,
     "path": "/producthunt/product/{id}/alternatives",
     "pathParams": [
       "id"
@@ -6839,6 +6872,7 @@ export const operations = {
     "formParams": [],
     "id": "producthunt-category-products",
     "method": "GET",
+    "paginatable": true,
     "path": "/producthunt/category/{slug}/products",
     "pathParams": [
       "slug"
@@ -6886,6 +6920,7 @@ export const operations = {
     "formParams": [],
     "id": "producthunt-customers",
     "method": "GET",
+    "paginatable": true,
     "path": "/producthunt/product/{id}/customers",
     "pathParams": [
       "id"
@@ -6924,9 +6959,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "producthunt-launches",
     "method": "GET",
+    "paginatable": true,
     "path": "/producthunt/product/{id}/launches",
     "pathParams": [
       "id"
@@ -6956,9 +6995,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "producthunt-leaderboard",
     "method": "GET",
+    "paginatable": true,
     "path": "/producthunt/leaderboard",
     "pathParams": [],
     "produces": [
@@ -7027,9 +7070,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "producthunt-makers",
     "method": "GET",
+    "paginatable": true,
     "path": "/producthunt/product/{id}/makers",
     "pathParams": [
       "id"
@@ -7099,6 +7146,7 @@ export const operations = {
     "formParams": [],
     "id": "producthunt-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/producthunt/search",
     "pathParams": [],
     "produces": [
@@ -7726,6 +7774,7 @@ export const operations = {
     "formParams": [],
     "id": "shopify-collection-products",
     "method": "GET",
+    "paginatable": true,
     "path": "/shopify/collections/{handle}/products",
     "pathParams": [
       "handle"
@@ -7764,6 +7813,7 @@ export const operations = {
     "formParams": [],
     "id": "shopify-collections",
     "method": "GET",
+    "paginatable": true,
     "path": "/shopify/collections",
     "pathParams": [],
     "produces": [
@@ -7828,6 +7878,7 @@ export const operations = {
     "formParams": [],
     "id": "shopify-pages",
     "method": "GET",
+    "paginatable": true,
     "path": "/shopify/pages",
     "pathParams": [],
     "produces": [
@@ -7934,6 +7985,7 @@ export const operations = {
     "formParams": [],
     "id": "shopify-products",
     "method": "GET",
+    "paginatable": true,
     "path": "/shopify/products",
     "pathParams": [],
     "produces": [
@@ -8156,6 +8208,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-album",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/album",
     "pathParams": [],
     "produces": [
@@ -8196,6 +8249,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-album-tracks",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/album/tracks",
     "pathParams": [],
     "produces": [
@@ -8236,6 +8290,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-albums-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/albums/search",
     "pathParams": [],
     "produces": [
@@ -8332,6 +8387,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-artist-albums",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/artist/albums",
     "pathParams": [],
     "produces": [
@@ -8442,6 +8498,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-artists-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/artists/search",
     "pathParams": [],
     "produces": [
@@ -8508,6 +8565,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-audiobook-chapters",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/audiobook/chapters",
     "pathParams": [],
     "produces": [
@@ -8548,6 +8606,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-audiobooks-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/audiobooks/search",
     "pathParams": [],
     "produces": [
@@ -8644,6 +8703,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-episodes-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/episodes/search",
     "pathParams": [],
     "produces": [
@@ -8805,6 +8865,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-playlist",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/playlist",
     "pathParams": [],
     "produces": [
@@ -8855,6 +8916,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-playlists-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/playlists/search",
     "pathParams": [],
     "produces": [
@@ -9086,6 +9148,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-podcasts-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify-podcasts/search",
     "pathParams": [],
     "produces": [
@@ -9187,6 +9250,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-podcasts-show-episodes",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify-podcasts/show/episodes",
     "pathParams": [],
     "produces": [
@@ -9327,6 +9391,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-profile-followers",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/profile/followers",
     "pathParams": [],
     "produces": [
@@ -9372,6 +9437,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-profile-playlists",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/profile/playlists",
     "pathParams": [],
     "produces": [
@@ -9417,6 +9483,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-profiles-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/profiles/search",
     "pathParams": [],
     "produces": [
@@ -9483,6 +9550,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/search",
     "pathParams": [],
     "produces": [
@@ -9559,6 +9627,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-section",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/section",
     "pathParams": [],
     "produces": [
@@ -9599,6 +9668,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-shows-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/shows/search",
     "pathParams": [],
     "produces": [
@@ -9740,6 +9810,7 @@ export const operations = {
     "formParams": [],
     "id": "spotify-tracks-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/spotify/tracks/search",
     "pathParams": [],
     "produces": [
@@ -9843,9 +9914,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "tiktok-challenge-list",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/hashtags",
     "pathParams": [],
     "produces": [
@@ -9917,6 +9992,7 @@ export const operations = {
     "formParams": [],
     "id": "tiktok-popular-trend-creator",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/popular-trend/creator",
     "pathParams": [],
     "produces": [
@@ -10012,9 +10088,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "tiktok-profile-post",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/posts",
     "pathParams": [],
     "produces": [
@@ -10053,9 +10133,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "tiktok-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/search",
     "pathParams": [],
     "produces": [
@@ -10089,9 +10173,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "tiktok-search-hashtag",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/search/hashtag",
     "pathParams": [],
     "produces": [
@@ -10125,9 +10213,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "tiktok-search-user",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/search/user",
     "pathParams": [],
     "produces": [
@@ -10252,6 +10344,7 @@ export const operations = {
     "formParams": [],
     "id": "tiktok-top-ads-list",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/top-ads/list",
     "pathParams": [],
     "produces": [
@@ -10414,6 +10507,7 @@ export const operations = {
     "formParams": [],
     "id": "tiktok-top-ads-recommend",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/top-ads/recommend",
     "pathParams": [],
     "produces": [
@@ -10469,6 +10563,7 @@ export const operations = {
     "formParams": [],
     "id": "tiktok-top-ads-spotlight",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/top-ads/spotlight",
     "pathParams": [],
     "produces": [
@@ -10545,9 +10640,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "cursor"
+    ],
     "formParams": [],
     "id": "tiktok-video-comments",
     "method": "GET",
+    "paginatable": true,
     "path": "/tiktok/comments",
     "pathParams": [],
     "produces": [
@@ -10657,6 +10756,7 @@ export const operations = {
     "formParams": [],
     "id": "tripadvisor-hotels",
     "method": "GET",
+    "paginatable": true,
     "path": "/tripadvisor/hotels",
     "pathParams": [],
     "produces": [
@@ -10774,6 +10874,7 @@ export const operations = {
     "formParams": [],
     "id": "tripadvisor-reviews",
     "method": "GET",
+    "paginatable": true,
     "path": "/tripadvisor/reviews",
     "pathParams": [],
     "produces": [
@@ -10845,6 +10946,7 @@ export const operations = {
     "formParams": [],
     "id": "tripadvisor-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/tripadvisor/search",
     "pathParams": [],
     "produces": [
@@ -11012,6 +11114,7 @@ export const operations = {
     "formParams": [],
     "id": "trustpilot-business-reviews",
     "method": "GET",
+    "paginatable": true,
     "path": "/trustpilot/business/{slug}/reviews",
     "pathParams": [
       "slug"
@@ -11074,6 +11177,7 @@ export const operations = {
     "formParams": [],
     "id": "trustpilot-business-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/trustpilot/business-units/search",
     "pathParams": [],
     "produces": [
@@ -11134,6 +11238,7 @@ export const operations = {
     "formParams": [],
     "id": "trustpilot-category",
     "method": "GET",
+    "paginatable": true,
     "path": "/trustpilot/category/{slug}",
     "pathParams": [
       "slug"
@@ -11469,9 +11574,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "start"
+    ],
     "formParams": [],
     "id": "yahoo-finance-calendar",
     "method": "GET",
+    "paginatable": true,
     "path": "/yahoo-finance/calendars/{type}",
     "pathParams": [
       "type"
@@ -11644,6 +11753,7 @@ export const operations = {
     "formParams": [],
     "id": "yahoo-finance-screener",
     "method": "GET",
+    "paginatable": true,
     "path": "/yahoo-finance/screener/{id}",
     "pathParams": [
       "id"
@@ -11941,6 +12051,7 @@ export const operations = {
     "formParams": [],
     "id": "yahoo-finance-ticker-earnings-dates",
     "method": "GET",
+    "paginatable": true,
     "path": "/yahoo-finance/ticker/{symbol}/earnings-dates",
     "pathParams": [
       "symbol"
@@ -12023,9 +12134,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "start"
+    ],
     "formParams": [],
     "id": "yahoo-finance-ticker-history",
     "method": "GET",
+    "paginatable": true,
     "path": "/yahoo-finance/ticker/{symbol}/history",
     "pathParams": [
       "symbol"
@@ -12317,9 +12432,13 @@ export const operations = {
     "consumes": [
       "application/json"
     ],
+    "cursorParams": [
+      "start"
+    ],
     "formParams": [],
     "id": "yahoo-finance-ticker-shares-full",
     "method": "GET",
+    "paginatable": true,
     "path": "/yahoo-finance/ticker/{symbol}/shares-full",
     "pathParams": [
       "symbol"
@@ -12925,6 +13044,7 @@ export const operations = {
     "formParams": [],
     "id": "zillow-search",
     "method": "GET",
+    "paginatable": true,
     "path": "/zillow/search",
     "pathParams": [],
     "produces": [