@crawlora-org/sdk 1.2.0-sdk.18 → 1.3.0-sdk.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crawlora-org/sdk",
-  "version": "1.2.0-sdk.18",
+  "version": "1.3.0-sdk.1",
   "description": "JavaScript and TypeScript SDK for the public Crawlora API.",
   "keywords": [
     "crawlora",

package/src/client.js

@@ -1,7 +1,7 @@
 import { operations, groups } from "./operations.js";
 
 const DEFAULT_BASE_URL = "https://api.crawlora.net/api/v1";
-export const VERSION = "1.2.0-sdk.18";
+export const VERSION = "1.3.0-sdk.1";
 const DEFAULT_USER_AGENT = `crawlora-js-sdk/${VERSION}`;
 
 export class CrawloraError extends Error {
@@ -18,6 +18,35 @@ export class CrawloraError extends Error {
   }
 }
 
+// Response status 4xx: the request was rejected by the API (bad params, auth,
+// not found). Usually not retryable.
+export class CrawloraClientError extends CrawloraError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "CrawloraClientError";
+  }
+}
+
+// Response status 5xx: the API failed to handle a valid request. Retryable.
+export class CrawloraServerError extends CrawloraError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "CrawloraServerError";
+  }
+}
+
+// Transport failure, timeout, or abort before a response was received.
+export class CrawloraNetworkError extends CrawloraError {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "CrawloraNetworkError";
+  }
+}
+
+function apiErrorClass(status) {
+  return status >= 500 ? CrawloraServerError : CrawloraClientError;
+}
+
 export class CrawloraClient {
   constructor(options = {}) {
     this.apiKey = options.apiKey || "";
@@ -98,12 +127,12 @@ export class CrawloraClient {
       });
     } catch (error) {
       if (options.signal?.aborted) {
-        throw new CrawloraError("Crawlora request aborted", { cause: error, retryable: false });
+        throw new CrawloraNetworkError("Crawlora request aborted", { cause: error, retryable: false });
       }
       if (timedOut) {
-        throw new CrawloraError("Crawlora request timed out", { cause: error, retryable: false });
+        throw new CrawloraNetworkError("Crawlora request timed out", { cause: error, retryable: false });
       }
-      throw new CrawloraError("Crawlora transport error", { cause: error });
+      throw new CrawloraNetworkError("Crawlora transport error", { cause: error });
     } finally {
       if (timeout) clearTimeout(timeout);
     }
@@ -113,7 +142,8 @@ export class CrawloraClient {
     if (!response.ok) {
       const code = parsed && typeof parsed === "object" ? parsed.code : undefined;
       const message = parsed && typeof parsed === "object" && parsed.msg ? parsed.msg : response.statusText;
-      throw new CrawloraError(message || `Crawlora request failed with status ${response.status}`, {
+      const ApiError = apiErrorClass(response.status);
+      throw new ApiError(message || `Crawlora request failed with status ${response.status}`, {
         status: response.status,
         code,
         body: parsed,
@@ -123,6 +153,50 @@ export class CrawloraClient {
     }
     return parsed;
   }
+
+  // Async iterator over pages of a paginated operation. Advances the numeric
+  // page/offset query parameter and stops when a page returns no data.
+  //   for await (const page of client.paginate("ebay-seller-feedback", { seller })) { ... }
+  async *paginate(operationId, params = {}, options = {}) {
+    const operation = operations[operationId];
+    if (!operation) {
+      throw new TypeError(`Unknown Crawlora operation: ${operationId}`);
+    }
+    const pageParam = options.pageParam || detectPageParam(operation);
+    if (!pageParam) {
+      throw new TypeError(`Operation ${operationId} has no page or offset query parameter to paginate`);
+    }
+    const step = options.step ?? 1;
+    const maxPages = options.maxPages ?? Infinity;
+    let pageValue = options.start ?? (pageParam === "offset" ? 0 : 1);
+    for (let i = 0; i < maxPages; i++) {
+      const response = await this.request(operationId, { ...params, [pageParam]: pageValue }, options);
+      yield response;
+      if (pageIsEmpty(response)) break;
+      pageValue += step;
+    }
+  }
+}
+
+const PAGE_PARAM_NAMES = ["page", "offset"];
+
+function detectPageParam(operation) {
+  for (const name of PAGE_PARAM_NAMES) {
+    if (operation.queryParams.some((parameter) => parameter.name === name)) return name;
+  }
+  return undefined;
+}
+
+function pageIsEmpty(response) {
+  if (response === undefined || response === null) return true;
+  let data = response;
+  if (typeof response === "object" && !Array.isArray(response) && "data" in response) {
+    data = response.data;
+  }
+  if (data === undefined || data === null) return true;
+  if (Array.isArray(data)) return data.length === 0;
+  if (typeof data === "object") return Object.keys(data).length === 0;
+  return !data;
 }
 
 function buildRequest(operation, baseUrl, params) {

package/src/index.d.ts

@@ -1,6 +1,7 @@
 import type {
   CrawloraGeneratedGroups,
   OperationId,
+  OperationParamsMap,
   OperationRequestArgs,
   OperationResponseMap
 } from "./types.js";
@@ -55,6 +56,24 @@ export class CrawloraError extends Error {
   retryable?: boolean;
 }
 
+// Thrown for 4xx API responses (request rejected by the API).
+export class CrawloraClientError extends CrawloraError {}
+// Thrown for 5xx API responses (API failed to handle a valid request).
+export class CrawloraServerError extends CrawloraError {}
+// Thrown for transport failures, timeouts, and aborts before a response.
+export class CrawloraNetworkError extends CrawloraError {}
+
+export interface CrawloraPaginateOptions extends CrawloraRequestOptions {
+  // Query parameter to advance. Auto-detected as "page" or "offset" when omitted.
+  pageParam?: string;
+  // First value for the page parameter. Defaults to 1 for "page", 0 for "offset".
+  start?: number;
+  // Amount added to the page parameter after each page. Defaults to 1.
+  step?: number;
+  // Maximum number of pages to fetch. Defaults to unbounded.
+  maxPages?: number;
+}
+
 export class CrawloraClient {
   constructor(options?: CrawloraClientOptions);
   request<I extends OperationId>(
@@ -65,6 +84,11 @@ export class CrawloraClient {
     operationId: I,
     ...args: OperationRequestArgs<I>
   ): Promise<OperationResponseMap[I]>;
+  paginate<I extends OperationId>(
+    operationId: I,
+    params?: OperationParamsMap[I],
+    options?: CrawloraPaginateOptions
+  ): AsyncGenerator<OperationResponseMap[I], void, unknown>;
   [group: string]: unknown;
 }
 
@@ -73,5 +97,6 @@ export interface CrawloraClient extends CrawloraGeneratedGroups {}
 export const operations: Record<string, OperationDefinition>;
 export const groups: Record<string, Record<string, string>>;
 export const operationCount: number;
+export const OperationIds: Readonly<Record<string, OperationId>>;
 export const VERSION: string;
 export * from "./types.js";

package/src/index.js

@@ -1,2 +1,9 @@
-export { CrawloraClient, CrawloraError, VERSION } from "./client.js";
-export { operations, groups, operationCount } from "./operations.js";
+export {
+  CrawloraClient,
+  CrawloraError,
+  CrawloraClientError,
+  CrawloraServerError,
+  CrawloraNetworkError,
+  VERSION
+} from "./client.js";
+export { operations, groups, operationCount, OperationIds } from "./operations.js";