@hyperbrowser/sdk 0.23.0 → 0.25.0

package/dist/services/scrape.d.ts

@@ -1,6 +1,27 @@
-import { ScrapeJobResponse, StartScrapeJobParams, StartScrapeJobResponse } from "../types/scrape";
+import { BatchScrapeJobResponse, GetBatchScrapeJobParams, ScrapeJobResponse, StartBatchScrapeJobParams, StartBatchScrapeJobResponse, StartScrapeJobParams, StartScrapeJobResponse } from "../types/scrape";
 import { BaseService } from "./base";
+export declare class BatchScrapeService extends BaseService {
+    /**
+     * Start a new batch scrape job
+     * @param params The parameters for the batch scrape job
+     */
+    start(params: StartBatchScrapeJobParams): Promise<StartBatchScrapeJobResponse>;
+    /**
+     * Get the status of a batch scrape job
+     * @param id The ID of the batch scrape job to get
+     * @param params Optional parameters to filter the batch scrape job
+     */
+    get(id: string, params?: GetBatchScrapeJobParams): Promise<BatchScrapeJobResponse>;
+    /**
+     * Start a batch scrape job and wait for it to complete
+     * @param params The parameters for the batch scrape job
+     * @param returnAllPages Whether to return all pages in the batch scrape job response
+     */
+    startAndWait(params: StartBatchScrapeJobParams, returnAllPages?: boolean): Promise<BatchScrapeJobResponse>;
+}
 export declare class ScrapeService extends BaseService {
+    readonly batch: BatchScrapeService;
+    constructor(apiKey: string, baseUrl: string, timeout: number);
     /**
      * Start a new scrape job
      * @param params The parameters for the scrape job

package/dist/services/scrape.js

@@ -1,10 +1,111 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ScrapeService = void 0;
+exports.ScrapeService = exports.BatchScrapeService = void 0;
 const base_1 = require("./base");
 const utils_1 = require("../utils");
 const client_1 = require("../client");
+class BatchScrapeService extends base_1.BaseService {
+    /**
+     * Start a new batch scrape job
+     * @param params The parameters for the batch scrape job
+     */
+    async start(params) {
+        try {
+            return await this.request("/scrape/batch", {
+                method: "POST",
+                body: JSON.stringify(params),
+            });
+        }
+        catch (error) {
+            if (error instanceof client_1.HyperbrowserError) {
+                throw error;
+            }
+            throw new client_1.HyperbrowserError("Failed to start batch scrape job", undefined);
+        }
+    }
+    /**
+     * Get the status of a batch scrape job
+     * @param id The ID of the batch scrape job to get
+     * @param params Optional parameters to filter the batch scrape job
+     */
+    async get(id, params) {
+        try {
+            return await this.request(`/scrape/batch/${id}`, undefined, {
+                page: params?.page,
+            });
+        }
+        catch (error) {
+            if (error instanceof client_1.HyperbrowserError) {
+                throw error;
+            }
+            throw new client_1.HyperbrowserError(`Failed to get batch scrape job ${id}`, undefined);
+        }
+    }
+    /**
+     * Start a batch scrape job and wait for it to complete
+     * @param params The parameters for the batch scrape job
+     * @param returnAllPages Whether to return all pages in the batch scrape job response
+     */
+    async startAndWait(params, returnAllPages = true) {
+        const job = await this.start(params);
+        const jobId = job.jobId;
+        if (!jobId) {
+            throw new client_1.HyperbrowserError("Failed to start batch scrape job, could not get job ID");
+        }
+        let jobResponse;
+        let failures = 0;
+        while (true) {
+            try {
+                jobResponse = await this.get(jobId);
+                if (jobResponse.status === "completed" || jobResponse.status === "failed") {
+                    break;
+                }
+                failures = 0;
+            }
+            catch (error) {
+                failures++;
+                if (failures >= 5) {
+                    throw new client_1.HyperbrowserError(`Failed to poll batch scrape job ${jobId} after 5 attempts: ${error}`);
+                }
+            }
+            await (0, utils_1.sleep)(2000);
+        }
+        if (!returnAllPages) {
+            return jobResponse;
+        }
+        failures = 0;
+        while (jobResponse.currentPageBatch < jobResponse.totalPageBatches) {
+            try {
+                const tmpJobResponse = await this.get(jobId, {
+                    page: jobResponse.currentPageBatch + 1,
+                    batchSize: 100,
+                });
+                if (tmpJobResponse.data) {
+                    jobResponse.data?.push(...tmpJobResponse.data);
+                }
+                jobResponse.currentPageBatch = tmpJobResponse.currentPageBatch;
+                jobResponse.totalScrapedPages = tmpJobResponse.totalScrapedPages;
+                jobResponse.totalPageBatches = tmpJobResponse.totalPageBatches;
+                jobResponse.batchSize = tmpJobResponse.batchSize;
+                failures = 0;
+            }
+            catch (error) {
+                failures++;
+                if (failures >= 5) {
+                    throw new client_1.HyperbrowserError(`Failed to get batch page ${jobResponse.currentPageBatch + 1} for job ${jobId} after 5 attempts: ${error}`);
+                }
+            }
+            await (0, utils_1.sleep)(500);
+        }
+        return jobResponse;
+    }
+}
+exports.BatchScrapeService = BatchScrapeService;
 class ScrapeService extends base_1.BaseService {
+    constructor(apiKey, baseUrl, timeout) {
+        super(apiKey, baseUrl, timeout);
+        this.batch = new BatchScrapeService(apiKey, baseUrl, timeout);
+    }
     /**
      * Start a new scrape job
      * @param params The parameters for the scrape job

package/dist/types/constants.d.ts

@@ -2,7 +2,9 @@ export type ScrapeFormat = "markdown" | "html" | "links" | "screenshot";
 export type ScrapeJobStatus = "pending" | "running" | "completed" | "failed";
 export type ExtractJobStatus = "pending" | "running" | "completed" | "failed";
 export type CrawlJobStatus = "pending" | "running" | "completed" | "failed";
+export type ScrapePageStatus = "completed" | "failed";
 export type CrawlPageStatus = "completed" | "failed";
+export type ScrapeWaitUntil = "load" | "domcontentloaded" | "networkidle";
 export type Country = "AD" | "AE" | "AF" | "AL" | "AM" | "AO" | "AR" | "AT" | "AU" | "AW" | "AZ" | "BA" | "BD" | "BE" | "BG" | "BH" | "BJ" | "BO" | "BR" | "BS" | "BT" | "BY" | "BZ" | "CA" | "CF" | "CH" | "CI" | "CL" | "CM" | "CN" | "CO" | "CR" | "CU" | "CY" | "CZ" | "DE" | "DJ" | "DK" | "DM" | "EC" | "EE" | "EG" | "ES" | "ET" | "EU" | "FI" | "FJ" | "FR" | "GB" | "GE" | "GH" | "GM" | "GR" | "HK" | "HN" | "HR" | "HT" | "HU" | "ID" | "IE" | "IL" | "IN" | "IQ" | "IR" | "IS" | "IT" | "JM" | "JO" | "JP" | "KE" | "KH" | "KR" | "KW" | "KZ" | "LB" | "LI" | "LR" | "LT" | "LU" | "LV" | "MA" | "MC" | "MD" | "ME" | "MG" | "MK" | "ML" | "MM" | "MN" | "MR" | "MT" | "MU" | "MV" | "MX" | "MY" | "MZ" | "NG" | "NL" | "NO" | "NZ" | "OM" | "PA" | "PE" | "PH" | "PK" | "PL" | "PR" | "PT" | "PY" | "QA" | "RANDOM_COUNTRY" | "RO" | "RS" | "RU" | "SA" | "SC" | "SD" | "SE" | "SG" | "SI" | "SK" | "SN" | "SS" | "TD" | "TG" | "TH" | "TM" | "TN" | "TR" | "TT" | "TW" | "UA" | "UG" | "US" | "UY" | "UZ" | "VE" | "VG" | "VN" | "YE" | "ZA" | "ZM" | "ZW" | "ad" | "ae" | "af" | "al" | "am" | "ao" | "ar" | "at" | "au" | "aw" | "az" | "ba" | "bd" | "be" | "bg" | "bh" | "bj" | "bo" | "br" | "bs" | "bt" | "by" | "bz" | "ca" | "cf" | "ch" | "ci" | "cl" | "cm" | "cn" | "co" | "cr" | "cu" | "cy" | "cz" | "de" | "dj" | "dk" | "dm" | "ec" | "ee" | "eg" | "es" | "et" | "eu" | "fi" | "fj" | "fr" | "gb" | "ge" | "gh" | "gm" | "gr" | "hk" | "hn" | "hr" | "ht" | "hu" | "id" | "ie" | "il" | "in" | "iq" | "ir" | "is" | "it" | "jm" | "jo" | "jp" | "ke" | "kh" | "kr" | "kw" | "kz" | "lb" | "li" | "lr" | "lt" | "lu" | "lv" | "ma" | "mc" | "md" | "me" | "mg" | "mk" | "ml" | "mm" | "mn" | "mr" | "mt" | "mu" | "mv" | "mx" | "my" | "mz" | "ng" | "nl" | "no" | "nz" | "om" | "pa" | "pe" | "ph" | "pk" | "pl" | "pr" | "pt" | "py" | "qa" | "ro" | "rs" | "ru" | "sa" | "sc" | "sd" | "se" | "sg" | "si" | "sk" | "sn" | "ss" | "td" | "tg" | "th" | "tm" | "tn" | "tr" | "tt" | "tw" | "ua" | "ug" | "us" | "uy" | "uz" | "ve" | "vg" | "vn" | "ye" | "za" | "zm" | "zw";
 export type OperatingSystem = "windows" | "android" | "macos" | "linux" | "ios";
 export type Platform = "chrome" | "firefox" | "safari" | "edge";

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
 export { HyperbrowserConfig } from "./config";
 export { StartCrawlJobParams, StartCrawlJobResponse, CrawledPage, CrawlJobResponse, GetCrawlJobParams, } from "./crawl";
-export { StartScrapeJobParams, StartScrapeJobResponse, ScrapeJobData, ScrapeJobResponse, } from "./scrape";
+export { StartScrapeJobParams, StartScrapeJobResponse, ScrapeJobData, ScrapeJobResponse, ScrapeOptions, } from "./scrape";
 export { BasicResponse, SessionStatus, Session, SessionDetail, SessionListParams, SessionListResponse, ScreenConfig, CreateSessionParams, } from "./session";
 export { ProfileResponse, CreateProfileResponse, ProfileListParams, ProfileListResponse, } from "./profile";
 export { CreateExtensionParams, CreateExtensionResponse, ListExtensionsResponse, } from "./extension";
-export { ScrapeJobStatus, CrawlJobStatus, Country, ISO639_1, OperatingSystem, Platform, } from "./constants";
+export { ScrapeJobStatus, CrawlJobStatus, Country, ISO639_1, OperatingSystem, Platform, ScrapeFormat, ScrapeWaitUntil, ScrapePageStatus, CrawlPageStatus, } from "./constants";

package/dist/types/scrape.d.ts

@@ -1,4 +1,4 @@
-import { ScrapeFormat, ScrapeJobStatus } from "./constants";
+import { ScrapeFormat, ScrapeJobStatus, ScrapePageStatus, ScrapeWaitUntil } from "./constants";
 import { CreateSessionParams } from "./session";
 export interface ScrapeOptions {
     formats?: ScrapeFormat[];
@@ -7,6 +7,7 @@ export interface ScrapeOptions {
     onlyMainContent?: boolean;
     waitFor?: number;
     timeout?: number;
+    waitUntil?: ScrapeWaitUntil;
 }
 export interface StartScrapeJobParams {
     url: string;
@@ -29,3 +30,35 @@ export interface ScrapeJobResponse {
     data?: ScrapeJobData;
     error?: string;
 }
+export interface StartBatchScrapeJobParams {
+    urls: string[];
+    sessionOptions?: CreateSessionParams;
+    scrapeOptions?: ScrapeOptions;
+}
+export interface ScrapedPage {
+    url: string;
+    status: ScrapePageStatus;
+    error?: string | null;
+    metadata?: Record<string, string | string[]>;
+    markdown?: string;
+    html?: string;
+    links?: string[];
+    screenshot?: string;
+}
+export interface GetBatchScrapeJobParams {
+    page?: number;
+    batchSize?: number;
+}
+export interface StartBatchScrapeJobResponse {
+    jobId: string;
+}
+export interface BatchScrapeJobResponse {
+    jobId: string;
+    status: ScrapeJobStatus;
+    data?: ScrapedPage[];
+    error?: string;
+    totalScrapedPages: number;
+    totalPageBatches: number;
+    currentPageBatch: number;
+    batchSize: number;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hyperbrowser/sdk",
-  "version": "0.23.0",
+  "version": "0.25.0",
   "description": "Node SDK for Hyperbrowser API",
   "author": "",
   "main": "dist/index.js",