@crawlee/browser-pool 4.0.0-beta.7 → 4.0.0-beta.70

package/remote-browser-provider.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Abstract base class for remote browser service providers.
+ *
+ * Implement this class to encapsulate the lifecycle of a remote browser session
+ * (creation, connection URL resolution, and cleanup). {@link RemoteBrowserPool}
+ * calls {@link connect} once per browser launch and {@link release} when the browser
+ * closes, crashes, the pool is destroyed, or the connection fails during launch.
+ *
+ * Pass the provider instance as the `endpoint` of a {@link RemoteBrowserPool}, then
+ * hand the pool to a crawler via its `browserPool` option:
+ *
+ * ```typescript
+ * const browserPool = new RemoteBrowserPool({
+ *     browserPlugins: [new PlaywrightPlugin(playwright.chromium)],
+ *     endpoint: new MyProvider(),
+ * });
+ *
+ * const crawler = new PlaywrightCrawler({ browserPool });
+ * ```
+ *
+ * **Example — simple static endpoint (e.g. Browserless):**
+ * ```typescript
+ * class BrowserlessProvider extends RemoteBrowserProvider {
+ *     maxOpenBrowsers = 2; // respect the service's concurrent session limit
+ *
+ *     async connect() {
+ *         return { url: `wss://production-sfo.browserless.io?token=${token}` };
+ *     }
+ * }
+ * ```
+ *
+ * **Example — session lifecycle with concurrency limit (e.g. Browserbase):**
+ * ```typescript
+ * class BrowserbaseProvider extends RemoteBrowserProvider<{ id: string }> {
+ *     maxOpenBrowsers = 2; // respect the service's concurrent session limit
+ *
+ *     async connect({ proxyUrl } = {}) {
+ *         const session = await createSession(apiKey, projectId, {
+ *             proxies: proxyUrl ? [{ type: 'external', server: proxyUrl }] : undefined,
+ *         });
+ *         return { url: session.connectUrl, context: { id: session.id } };
+ *     }
+ *
+ *     async release(context: { id: string }) {
+ *         await releaseSession(apiKey, context.id);
+ *     }
+ * }
+ * ```
+ */
+export declare abstract class RemoteBrowserProvider<TContext extends Record<string, unknown> = Record<string, unknown>> {
+    /**
+     * Maximum number of browsers that can be open at the same time.
+     * Set this to your remote service's concurrent session limit to avoid 429 errors.
+     */
+    maxOpenBrowsers?: number;
+    /**
+     * Called once per browser launch. Return the WebSocket/CDP endpoint URL
+     * and an optional `context` object that will be passed back to {@link release}.
+     *
+     * @param options.proxyUrl - The proxy URL resolved by Crawlee's proxy configuration
+     * for this browser session. Pass it to your remote service's proxy API if supported.
+     */
+    abstract connect(options?: {
+        proxyUrl?: string;
+    }): Promise<{
+        url: string;
+        context?: TContext;
+    }> | {
+        url: string;
+        context?: TContext;
+    };
+    /**
+     * Called when the browser closes, crashes, the pool is destroyed, or the
+     * connection fails right after {@link connect} succeeds.
+     * Override this to clean up remote sessions, release API resources, etc.
+     *
+     * Errors thrown here are caught and logged as warnings — they never crash the crawler.
+     * Safe to assume this is called at most once per {@link connect} call.
+     *
+     * @param _context The same `context` object returned by {@link connect}.
+     */
+    release(_context: TContext): Promise<void>;
+}
+//# sourceMappingURL=remote-browser-provider.d.ts.map

package/remote-browser-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-browser-provider.d.ts","sourceRoot":"","sources":["../src/remote-browser-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,8BAAsB,qBAAqB,CAAC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAC1G;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC,GAAG;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,QAAQ,CAAA;KAAE;IAEtF;;;;;;;;;OASG;IACG,OAAO,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;CACnD"}

package/remote-browser-provider.js

@@ -0,0 +1,68 @@
+/**
+ * Abstract base class for remote browser service providers.
+ *
+ * Implement this class to encapsulate the lifecycle of a remote browser session
+ * (creation, connection URL resolution, and cleanup). {@link RemoteBrowserPool}
+ * calls {@link connect} once per browser launch and {@link release} when the browser
+ * closes, crashes, the pool is destroyed, or the connection fails during launch.
+ *
+ * Pass the provider instance as the `endpoint` of a {@link RemoteBrowserPool}, then
+ * hand the pool to a crawler via its `browserPool` option:
+ *
+ * ```typescript
+ * const browserPool = new RemoteBrowserPool({
+ *     browserPlugins: [new PlaywrightPlugin(playwright.chromium)],
+ *     endpoint: new MyProvider(),
+ * });
+ *
+ * const crawler = new PlaywrightCrawler({ browserPool });
+ * ```
+ *
+ * **Example — simple static endpoint (e.g. Browserless):**
+ * ```typescript
+ * class BrowserlessProvider extends RemoteBrowserProvider {
+ *     maxOpenBrowsers = 2; // respect the service's concurrent session limit
+ *
+ *     async connect() {
+ *         return { url: `wss://production-sfo.browserless.io?token=${token}` };
+ *     }
+ * }
+ * ```
+ *
+ * **Example — session lifecycle with concurrency limit (e.g. Browserbase):**
+ * ```typescript
+ * class BrowserbaseProvider extends RemoteBrowserProvider<{ id: string }> {
+ *     maxOpenBrowsers = 2; // respect the service's concurrent session limit
+ *
+ *     async connect({ proxyUrl } = {}) {
+ *         const session = await createSession(apiKey, projectId, {
+ *             proxies: proxyUrl ? [{ type: 'external', server: proxyUrl }] : undefined,
+ *         });
+ *         return { url: session.connectUrl, context: { id: session.id } };
+ *     }
+ *
+ *     async release(context: { id: string }) {
+ *         await releaseSession(apiKey, context.id);
+ *     }
+ * }
+ * ```
+ */
+export class RemoteBrowserProvider {
+    /**
+     * Maximum number of browsers that can be open at the same time.
+     * Set this to your remote service's concurrent session limit to avoid 429 errors.
+     */
+    maxOpenBrowsers;
+    /**
+     * Called when the browser closes, crashes, the pool is destroyed, or the
+     * connection fails right after {@link connect} succeeds.
+     * Override this to clean up remote sessions, release API resources, etc.
+     *
+     * Errors thrown here are caught and logged as warnings — they never crash the crawler.
+     * Safe to assume this is called at most once per {@link connect} call.
+     *
+     * @param _context The same `context` object returned by {@link connect}.
+     */
+    async release(_context) { }
+}
+//# sourceMappingURL=remote-browser-provider.js.map

package/remote-browser-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-browser-provider.js","sourceRoot":"","sources":["../src/remote-browser-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,MAAM,OAAgB,qBAAqB;IACvC;;;OAGG;IACH,eAAe,CAAU;IAazB;;;;;;;;;OASG;IACH,KAAK,CAAC,OAAO,CAAC,QAAkB,IAAkB,CAAC;CACtD"}

package/utils.d.ts

@@ -3,6 +3,13 @@ import type { PlaywrightPlugin } from './playwright/playwright-plugin.js';
 import type { PuppeteerPlugin } from './puppeteer/puppeteer-plugin.js';
 export type UnwrapPromise<T> = T extends PromiseLike<infer R> ? UnwrapPromise<R> : T;
 export declare function noop(..._args: unknown[]): void;
+/**
+ * Strips secrets from a URL so it can be safely included in logs and error messages. Removes userinfo
+ * credentials and the entire query string and fragment — remote browser services routinely carry tokens
+ * there (e.g. Browserless `?token=…`), and we can't tell which params are sensitive. Keeps the
+ * protocol, host, port, and path, which are enough to diagnose connection failures.
+ */
+export declare function sanitizeEndpointForLog(endpoint: string): string;
 /**
  * This is required when using optional dependencies.
  * Importing a type gives `any`, but `Parameters<any>` gives `unknown[]` instead of `any`

package/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAEvE,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAErF,wBAAgB,IAAI,CAAC,GAAG,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,CAAG;AAElD;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,GAAG,IAAI,OAAO,EAAE,SAAS,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;AAEpH,MAAM,MAAM,uBAAuB,CAE/B,KAAK,SAAS,SAAS,OAAO,EAAE,EAEhC,MAAM,SAAS,aAAa,EAAE,GAAG,EAAE,IACnC,KAAK,SAAS,SAAS,CAAC,MAAM,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,GAE1F,UAAU,SAAS,gBAAgB,GAE/B,uBAAuB,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,EAAE,gBAAgB,CAAC,CAAC,GAE5D,UAAU,SAAS,eAAe,GAEhC,uBAAuB,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,EAAE,eAAe,CAAC,CAAC,GAE3D,KAAK,GAEX,KAAK,SAAS,EAAE,GAEd,MAAM,GAEN,KAAK,SAAS,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,GAEhC;IAAC,CAAC;CAAC,SAAS,CAAC,eAAe,GAAG,gBAAgB,CAAC,GAE5C,CAAC,EAAE,GAEH,KAAK,GAET,MAAM,CAAC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAEvE,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAErF,wBAAgB,IAAI,CAAC,GAAG,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,CAAG;AAElD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAW/D;AAED;;;GAGG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,GAAG,IAAI,OAAO,EAAE,SAAS,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;AAEpH,MAAM,MAAM,uBAAuB,CAE/B,KAAK,SAAS,SAAS,OAAO,EAAE,EAEhC,MAAM,SAAS,aAAa,EAAE,GAAG,EAAE,IACnC,KAAK,SAAS,SAAS,CAAC,MAAM,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,GAE1F,UAAU,SAAS,gBAAgB,GAE/B,uBAAuB,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,EAAE,gBAAgB,CAAC,CAAC,GAE5D,UAAU,SAAS,eAAe,GAEhC,uBAAuB,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,EAAE,eAAe,CAAC,CAAC,GAE3D,KAAK,GAEX,KAAK,SAAS,EAAE,GAEd,MAAM,GAEN,KAAK,SAAS,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,GAEhC;IAAC,CAAC;CAAC,SAAS,CAAC,eAAe,GAAG,gBAAgB,CAAC,GAE5C,CAAC,EAAE,GAEH,KAAK,GAET,MAAM,CAAC"}

package/utils.js

@@ -1,2 +1,21 @@
 export function noop(..._args) { }
+/**
+ * Strips secrets from a URL so it can be safely included in logs and error messages. Removes userinfo
+ * credentials and the entire query string and fragment — remote browser services routinely carry tokens
+ * there (e.g. Browserless `?token=…`), and we can't tell which params are sensitive. Keeps the
+ * protocol, host, port, and path, which are enough to diagnose connection failures.
+ */
+export function sanitizeEndpointForLog(endpoint) {
+    try {
+        const url = new URL(endpoint);
+        url.username = '';
+        url.password = '';
+        url.search = '';
+        url.hash = '';
+        return url.toString();
+    }
+    catch {
+        return '<invalid URL>';
+    }
+}
 //# sourceMappingURL=utils.js.map

package/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAMA,MAAM,UAAU,IAAI,CAAC,GAAG,KAAgB,IAAS,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAMA,MAAM,UAAU,IAAI,CAAC,GAAG,KAAgB,IAAS,CAAC;AAElD;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB;IACnD,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,GAAG,CAAC,QAAQ,GAAG,EAAE,CAAC;QAClB,GAAG,CAAC,QAAQ,GAAG,EAAE,CAAC;QAClB,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC;QAChB,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;QACd,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,eAAe,CAAC;IAC3B,CAAC;AACL,CAAC"}

package/logger.d.ts

@@ -1,3 +0,0 @@
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare const log: import("@apify/log").Log;
-//# sourceMappingURL=logger.d.ts.map

package/logger.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,GAAG,0BAEd,CAAC"}

package/logger.js

@@ -1,5 +0,0 @@
-import defaultLog from '@apify/log';
-export const log = defaultLog.child({
-    prefix: 'BrowserPool',
-});
-//# sourceMappingURL=logger.js.map

package/logger.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,YAAY,CAAC;AAEpC,MAAM,CAAC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC;IAChC,MAAM,EAAE,aAAa;CACxB,CAAC,CAAC"}