@crawlora-org/sdk 1.2.0-sdk.19 → 1.4.0-sdk.1

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## v1.4.0-sdk.1
+
+- Configurable retries: `maxRetryDelay`, `retryStatuses`, and an `isRetryable`
+  predicate; added an `onRetry` hook, opt-in `requestId` (x-request-id, also on
+  `error.requestId`), and a `logger` event sink.
+- Pagination: cursor/token mode (`cursorParam` + `nextCursor`) and a
+  `paginateItems` per-item async iterator; operation metadata now exposes
+  `paginatable` and `cursorParams`.
+- Streaming: `responseType: "stream"` returns the raw `Response` on success.
+- Config: `CRAWLORA_API_KEY` / `CRAWLORA_BASE_URL` environment fallback.
+
+## v1.3.0-sdk.1
+
+- Added `CrawloraClientError`, `CrawloraServerError`, and `CrawloraNetworkError`
+  subclasses of `CrawloraError` so callers can branch on 4xx vs 5xx vs transport
+  failures.
+- Added `client.paginate(operationId, params, options?)`, an async iterator that
+  advances the page/offset query parameter and stops on an empty page.
+- Added the generated `OperationIds` map for typo-safe dynamic operation ids,
+  e.g. `client.request(OperationIds.BingSearch, { q: "coffee" })`.
+- The generator now shares a single language-neutral core with the Go and Python
+  SDKs; generated output is unchanged.
+
 ## v1.2.0-sdk.19
 
 - Regenerated the public SDK contract with the promoted Shopify endpoint family.

package/README.md

@@ -142,6 +142,23 @@ The error includes `status`, optional API `code`, parsed `body`, response
 headers, capped at 30 seconds. Externally aborted requests fail with
 `Crawlora request aborted` and are not retried.
 
+`CrawloraError` has three subclasses so you can branch on the failure kind:
+`CrawloraClientError` (4xx, request rejected), `CrawloraServerError` (5xx), and
+`CrawloraNetworkError` (transport failure, timeout, or abort before a response).
+
+## Pagination
+
+`client.paginate` is an async iterator that advances the page/offset query
+parameter and stops when a page returns no data:
+
+```js
+for await (const page of crawlora.paginate("ebay-seller-feedback", { seller: "acme" })) {
+  for (const review of page.data) console.log(review);
+}
+```
+
+Override detection with `{ pageParam, start, step, maxPages }`.
+
 ## Examples
 
 Runnable examples live under `examples/` and skip cleanly when required

package/docs/recipes.md

@@ -74,6 +74,51 @@ try {
 }
 ```
 
+## Custom Retries And Observability
+
+```js
+const crawlora = new CrawloraClient({
+  retries: 3,
+  maxRetryDelay: 10000,
+  retryStatuses: [429, 503],                       // or:
+  isRetryable: (status, error) => status >= 500,
+  onRetry: (attempt, error, delay) => console.warn(`retry ${attempt} after ${delay}ms`, error.status),
+  requestId: true,                                  // sets x-request-id; available as error.requestId
+  logger: (event) => console.debug(event)
+});
+```
+
+Branch on `CrawloraClientError` (4xx), `CrawloraServerError` (5xx), and
+`CrawloraNetworkError` (transport).
+
+## Pagination
+
+```js
+// page/offset (auto-detected)
+for await (const page of crawlora.paginate("ebay-seller-feedback", { seller: "acme" })) { /* ... */ }
+
+// per-item iteration
+for await (const item of crawlora.paginateItems("ebay-seller-feedback", { seller: "acme" })) { /* ... */ }
+
+// cursor/token pagination
+for await (const page of crawlora.paginate("producthunt-leaderboard", {}, {
+  cursorParam: "cursor",
+  nextCursor: (p) => p.next_cursor
+})) { /* ... */ }
+```
+
+## Streaming Responses
+
+```js
+const res = await crawlora.request("bing-search", { q: "coffee" }, { responseType: "stream" });
+for await (const chunk of res.body) { /* ... */ }
+```
+
+## Environment Variables
+
+`CRAWLORA_API_KEY` and `CRAWLORA_BASE_URL` are used when not set explicitly
+(precedence: option > env > default).
+
 ## Optional Live Smoke Tests
 
 ```sh

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crawlora-org/sdk",
-  "version": "1.2.0-sdk.19",
+  "version": "1.4.0-sdk.1",
   "description": "JavaScript and TypeScript SDK for the public Crawlora API.",
   "keywords": [
     "crawlora",

package/src/client.js

@@ -1,11 +1,11 @@
 import { operations, groups } from "./operations.js";
 
 const DEFAULT_BASE_URL = "https://api.crawlora.net/api/v1";
-export const VERSION = "1.2.0-sdk.19";
+export const VERSION = "1.4.0-sdk.1";
 const DEFAULT_USER_AGENT = `crawlora-js-sdk/${VERSION}`;
 
 export class CrawloraError extends Error {
-  constructor(message, { status = 0, code, body, headers = {}, response, cause, retryable } = {}) {
+  constructor(message, { status = 0, code, body, headers = {}, response, cause, retryable, requestId } = {}) {
     super(message);
     this.name = "CrawloraError";
     this.status = status;
@@ -15,17 +15,54 @@ export class CrawloraError extends Error {
     this.response = response;
     this.cause = cause;
     this.retryable = retryable;
+    this.requestId = requestId;
   }
 }
 
+// Response status 4xx: the request was rejected by the API (bad params, auth,
+// not found). Usually not retryable.
+export class CrawloraClientError extends CrawloraError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "CrawloraClientError";
+  }
+}
+
+// Response status 5xx: the API failed to handle a valid request. Retryable.
+export class CrawloraServerError extends CrawloraError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "CrawloraServerError";
+  }
+}
+
+// Transport failure, timeout, or abort before a response was received.
+export class CrawloraNetworkError extends CrawloraError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "CrawloraNetworkError";
+  }
+}
+
+function apiErrorClass(status) {
+  return status >= 500 ? CrawloraServerError : CrawloraClientError;
+}
+
 export class CrawloraClient {
   constructor(options = {}) {
-    this.apiKey = options.apiKey || "";
+    // Precedence: explicit option > environment variable > default.
+    this.apiKey = options.apiKey || envVar("CRAWLORA_API_KEY") || "";
     this.jwtToken = options.jwtToken || "";
-    this.baseUrl = (options.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, "");
+    this.baseUrl = (options.baseUrl || envVar("CRAWLORA_BASE_URL") || DEFAULT_BASE_URL).replace(/\/+$/, "");
     this.timeout = options.timeout ?? 30000;
     this.retries = normalizeNonNegativeInteger(options.retries ?? 0);
     this.retryDelay = normalizeNonNegativeNumber(options.retryDelay ?? 250);
+    this.maxRetryDelay = normalizeNonNegativeNumber(options.maxRetryDelay ?? 30000);
+    this.retryStatuses = options.retryStatuses ? new Set(options.retryStatuses) : null;
+    this.isRetryable = typeof options.isRetryable === "function" ? options.isRetryable : null;
+    this.onRetry = typeof options.onRetry === "function" ? options.onRetry : null;
+    this.requestId = options.requestId === true;
+    this.logger = typeof options.logger === "function" ? options.logger : null;
     this.headers = { ...(options.headers || {}) };
     this.userAgent = options.userAgent === false ? "" : (options.userAgent || DEFAULT_USER_AGENT);
     this.fetch = options.fetch || globalThis.fetch;
@@ -55,21 +92,40 @@ export class CrawloraClient {
     params = params ?? {};
     options = options ?? {};
     const responseType = validateResponseType(options.responseType || "auto");
+    this.#log({ event: "request", operation: operationId });
 
     let attempt = 0;
     for (;;) {
       try {
         return await this.#send(operation, params, options, responseType);
       } catch (error) {
-        if (!(error instanceof CrawloraError) || error.retryable === false || attempt >= this.retries || !shouldRetry(error.status)) {
+        if (!(error instanceof CrawloraError) || error.retryable === false || attempt >= this.retries || !this.#isRetryable(error.status, error)) {
           throw error;
         }
         attempt++;
-        await sleep(retryDelayForError(error, this.retryDelay, attempt), options.signal);
+        const delay = this.#retryDelayFor(error, attempt);
+        this.#log({ event: "retry", operation: operationId, attempt, status: error.status, delay });
+        if (this.onRetry) this.onRetry(attempt, error, delay);
+        await sleep(delay, options.signal);
       }
     }
   }
 
+  #isRetryable(status, error) {
+    if (this.isRetryable) return !!this.isRetryable(status, error);
+    if (this.retryStatuses) return status === 0 || this.retryStatuses.has(status);
+    return shouldRetry(status);
+  }
+
+  #retryDelayFor(error, attempt) {
+    const retryAfter = parseRetryAfter(error.headers, this.maxRetryDelay);
+    return retryAfter ?? retryDelay(this.retryDelay, attempt);
+  }
+
+  #log(event) {
+    if (this.logger) this.logger(event);
+  }
+
   async #send(operation, params, options, responseType) {
     const { url, body, bodyHeaders } = buildRequest(operation, this.baseUrl, params);
     const headers = mergeHeaders(
@@ -79,6 +135,7 @@ export class CrawloraClient {
       bodyHeaders,
       options.headers
     );
+    const requestId = this.requestId ? ensureRequestId(headers) : (headerValue(headers, "x-request-id") || undefined);
 
     const controller = new AbortController();
     const timeoutMs = options.timeout ?? this.timeout;
@@ -98,31 +155,117 @@ export class CrawloraClient {
       });
     } catch (error) {
       if (options.signal?.aborted) {
-        throw new CrawloraError("Crawlora request aborted", { cause: error, retryable: false });
+        throw new CrawloraNetworkError("Crawlora request aborted", { cause: error, retryable: false, requestId });
       }
       if (timedOut) {
-        throw new CrawloraError("Crawlora request timed out", { cause: error, retryable: false });
+        throw new CrawloraNetworkError("Crawlora request timed out", { cause: error, retryable: false, requestId });
       }
-      throw new CrawloraError("Crawlora transport error", { cause: error });
+      throw new CrawloraNetworkError("Crawlora transport error", { cause: error, requestId });
     } finally {
       if (timeout) clearTimeout(timeout);
     }
 
     const headersObject = responseHeaders(response);
-    const parsed = await parseResponse(response, responseType, headersObject);
+    // Streaming success returns the raw Response; the caller reads response.body.
+    if (responseType === "stream" && response.ok) {
+      return response;
+    }
+    const parsed = await parseResponse(response, responseType === "stream" ? "auto" : responseType, headersObject, requestId);
     if (!response.ok) {
       const code = parsed && typeof parsed === "object" ? parsed.code : undefined;
       const message = parsed && typeof parsed === "object" && parsed.msg ? parsed.msg : response.statusText;
-      throw new CrawloraError(message || `Crawlora request failed with status ${response.status}`, {
+      const ApiError = apiErrorClass(response.status);
+      throw new ApiError(message || `Crawlora request failed with status ${response.status}`, {
         status: response.status,
         code,
         body: parsed,
         headers: headersObject,
-        response
+        response,
+        requestId
       });
     }
     return parsed;
   }
+
+  // Async iterator over pages of a paginated operation. Numeric mode advances
+  // the page/offset query parameter and stops on an empty page; cursor mode
+  // (cursorParam + nextCursor) sends the cursor and stops when nextCursor is falsy.
+  //   for await (const page of client.paginate("ebay-seller-feedback", { seller })) { ... }
+  async *paginate(operationId, params = {}, options = {}) {
+    const operation = operations[operationId];
+    if (!operation) {
+      throw new TypeError(`Unknown Crawlora operation: ${operationId}`);
+    }
+    const maxPages = options.maxPages ?? Infinity;
+
+    if (options.cursorParam || options.nextCursor) {
+      if (!(options.cursorParam && options.nextCursor)) {
+        throw new TypeError("cursor pagination requires both cursorParam and nextCursor");
+      }
+      if (!operation.queryParams.some((parameter) => parameter.name === options.cursorParam)) {
+        throw new TypeError(`cursorParam ${options.cursorParam} is not a query parameter of ${operationId}`);
+      }
+      let cursor = options.start;
+      for (let i = 0; i < maxPages; i++) {
+        const pageParams = { ...params };
+        if (cursor !== undefined && cursor !== null) pageParams[options.cursorParam] = cursor;
+        const response = await this.request(operationId, pageParams, options);
+        yield response;
+        cursor = options.nextCursor(response);
+        if (!cursor) break;
+      }
+      return;
+    }
+
+    const pageParam = options.pageParam || detectPageParam(operation);
+    if (!pageParam) {
+      throw new TypeError(`Operation ${operationId} has no page or offset query parameter to paginate`);
+    }
+    const step = options.step ?? 1;
+    let pageValue = options.start ?? (pageParam === "offset" ? 0 : 1);
+    for (let i = 0; i < maxPages; i++) {
+      const response = await this.request(operationId, { ...params, [pageParam]: pageValue }, options);
+      yield response;
+      if (pageIsEmpty(response)) break;
+      pageValue += step;
+    }
+  }
+
+  // Async iterator over individual items across pages. `items` extracts the list
+  // from a page (default: the Crawlora `data` array).
+  async *paginateItems(operationId, params = {}, options = {}) {
+    const extract = options.items || defaultItems;
+    for await (const page of this.paginate(operationId, params, options)) {
+      for (const item of extract(page)) yield item;
+    }
+  }
+}
+
+const PAGE_PARAM_NAMES = ["page", "offset"];
+
+function defaultItems(response) {
+  if (response && typeof response === "object" && Array.isArray(response.data)) return response.data;
+  if (Array.isArray(response)) return response;
+  return [];
+}
+
+function detectPageParam(operation) {
+  for (const name of PAGE_PARAM_NAMES) {
+    if (operation.queryParams.some((parameter) => parameter.name === name)) return name;
+  }
+  return undefined;
+}
+
+function pageIsEmpty(response) {
+  if (response === undefined || response === null) return true;
+  let data = response;
+  if (typeof response === "object" && !Array.isArray(response) && "data" in response) {
+    data = response.data;
+  }
+  if (data === undefined || data === null) return true;
+  if (Array.isArray(data)) return data.length === 0;
+  if (typeof data === "object") return Object.keys(data).length === 0;
+  return !data;
 }
 
 function buildRequest(operation, baseUrl, params) {
@@ -242,13 +385,13 @@ function composeSignal(controller, signal) {
 }
 
 function validateResponseType(responseType) {
-  if (responseType === "auto" || responseType === "json" || responseType === "text") {
+  if (responseType === "auto" || responseType === "json" || responseType === "text" || responseType === "stream") {
     return responseType;
   }
-  throw new TypeError("Invalid responseType: expected one of auto, json, text");
+  throw new TypeError("Invalid responseType: expected one of auto, json, text, stream");
 }
 
-async function parseResponse(response, responseType, headers) {
+async function parseResponse(response, responseType, headers, requestId) {
   if (responseType === "text") return response.text();
   const contentType = response.headers.get("content-type") || "";
   if (responseType === "json" || contentType.toLowerCase().includes("application/json")) {
@@ -261,7 +404,8 @@ async function parseResponse(response, responseType, headers) {
         body: text,
         headers,
         response,
-        cause: error
+        cause: error,
+        requestId
       });
     }
   }
@@ -279,25 +423,33 @@ function retryDelay(baseDelay, attempt) {
   return delay + jitter;
 }
 
-function retryDelayForError(error, baseDelay, attempt) {
-  return parseRetryAfter(error.headers) ?? retryDelay(baseDelay, attempt);
-}
-
-function parseRetryAfter(headers) {
+function parseRetryAfter(headers, cap = 30000) {
   const value = headerValue(headers, "retry-after");
   if (!value) return undefined;
   const seconds = Number(value);
   if (Number.isFinite(seconds) && seconds > 0) {
-    return Math.min(seconds * 1000, 30000);
+    return Math.min(seconds * 1000, cap);
   }
   const date = Date.parse(value);
   if (Number.isFinite(date)) {
     const delay = date - Date.now();
-    if (delay > 0) return Math.min(delay, 30000);
+    if (delay > 0) return Math.min(delay, cap);
   }
   return undefined;
 }
 
+function envVar(name) {
+  return (typeof process !== "undefined" && process.env && process.env[name]) || undefined;
+}
+
+function ensureRequestId(headers) {
+  const existing = headerValue(headers, "x-request-id");
+  if (existing) return existing;
+  const id = (typeof crypto !== "undefined" && crypto.randomUUID) ? crypto.randomUUID() : `req-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+  headers["x-request-id"] = id;
+  return id;
+}
+
 function headerValue(headers, name) {
   for (const [key, value] of Object.entries(headers || {})) {
     if (key.toLowerCase() === name.toLowerCase()) return value;

package/src/index.d.ts

@@ -1,12 +1,15 @@
 import type {
   CrawloraGeneratedGroups,
   OperationId,
+  OperationParamsMap,
   OperationRequestArgs,
   OperationResponseMap
 } from "./types.js";
 
 export type CrawloraParams = Record<string, unknown>;
 
+export type CrawloraLogEvent = { event: string; [key: string]: unknown };
+
 export interface CrawloraClientOptions {
   apiKey?: string;
   jwtToken?: string;
@@ -14,6 +17,18 @@ export interface CrawloraClientOptions {
   timeout?: number;
   retries?: number;
   retryDelay?: number;
+  /** Cap on backoff and Retry-After delays in ms (default 30000). */
+  maxRetryDelay?: number;
+  /** Override the retryable HTTP status set (network status 0 stays retryable). */
+  retryStatuses?: Iterable<number>;
+  /** Full predicate; supersedes retryStatuses when set. */
+  isRetryable?: (status: number, error: CrawloraError) => boolean;
+  /** Called before each retry sleep with (attempt, error, delayMs). */
+  onRetry?: (attempt: number, error: CrawloraError, delay: number) => void;
+  /** Generate an x-request-id header when absent. */
+  requestId?: boolean;
+  /** Structured event sink (request/retry); never logs on its own. */
+  logger?: (event: CrawloraLogEvent) => void;
   headers?: Record<string, string>;
   userAgent?: string | false;
   fetch?: typeof globalThis.fetch;
@@ -21,7 +36,7 @@ export interface CrawloraClientOptions {
 
 export interface CrawloraRequestOptions {
   headers?: Record<string, string>;
-  responseType?: "auto" | "json" | "text";
+  responseType?: "auto" | "json" | "text" | "stream";
   timeout?: number;
   signal?: AbortSignal;
 }
@@ -43,6 +58,8 @@ export interface OperationDefinition {
   consumes: string[];
   produces: string[];
   security: string[];
+  paginatable?: boolean;
+  cursorParams?: string[];
 }
 
 export class CrawloraError extends Error {
@@ -53,6 +70,34 @@ export class CrawloraError extends Error {
   response?: Response;
   cause?: unknown;
   retryable?: boolean;
+  requestId?: string;
+}
+
+// Thrown for 4xx API responses (request rejected by the API).
+export class CrawloraClientError extends CrawloraError {}
+// Thrown for 5xx API responses (API failed to handle a valid request).
+export class CrawloraServerError extends CrawloraError {}
+// Thrown for transport failures, timeouts, and aborts before a response.
+export class CrawloraNetworkError extends CrawloraError {}
+
+export interface CrawloraPaginateOptions extends CrawloraRequestOptions {
+  // Numeric query parameter to advance. Auto-detected as "page" or "offset".
+  pageParam?: string;
+  // Cursor/token query parameter (cursor mode). Requires nextCursor.
+  cursorParam?: string;
+  // Extracts the next cursor from a page; iteration stops when it returns falsy.
+  nextCursor?: (page: unknown) => unknown;
+  // First page value (numeric) or initial cursor value.
+  start?: unknown;
+  // Amount added to the page parameter after each page. Defaults to 1.
+  step?: number;
+  // Maximum number of pages to fetch. Defaults to unbounded.
+  maxPages?: number;
+}
+
+export interface CrawloraPaginateItemsOptions extends CrawloraPaginateOptions {
+  // Extracts the item list from a page (default: the `data` array).
+  items?: (page: unknown) => Iterable<unknown>;
 }
 
 export class CrawloraClient {
@@ -65,6 +110,16 @@ export class CrawloraClient {
     operationId: I,
     ...args: OperationRequestArgs<I>
   ): Promise<OperationResponseMap[I]>;
+  paginate<I extends OperationId>(
+    operationId: I,
+    params?: OperationParamsMap[I],
+    options?: CrawloraPaginateOptions
+  ): AsyncGenerator<OperationResponseMap[I], void, unknown>;
+  paginateItems<I extends OperationId>(
+    operationId: I,
+    params?: OperationParamsMap[I],
+    options?: CrawloraPaginateItemsOptions
+  ): AsyncGenerator<unknown, void, unknown>;
   [group: string]: unknown;
 }
 
@@ -73,5 +128,6 @@ export interface CrawloraClient extends CrawloraGeneratedGroups {}
 export const operations: Record<string, OperationDefinition>;
 export const groups: Record<string, Record<string, string>>;
 export const operationCount: number;
+export const OperationIds: Readonly<Record<string, OperationId>>;
 export const VERSION: string;
 export * from "./types.js";

package/src/index.js

@@ -1,2 +1,9 @@
-export { CrawloraClient, CrawloraError, VERSION } from "./client.js";
-export { operations, groups, operationCount } from "./operations.js";
+export {
+  CrawloraClient,
+  CrawloraError,
+  CrawloraClientError,
+  CrawloraServerError,
+  CrawloraNetworkError,
+  VERSION
+} from "./client.js";
+export { operations, groups, operationCount, OperationIds } from "./operations.js";