@brightdata/cli 0.1.3 → 0.1.5

package/dist/browser/daemon.js

@@ -0,0 +1,645 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.start_daemon_from_env = exports.parse_idle_timeout = exports.normalize_session_name = exports.get_port_for_session = exports.get_daemon_transport = exports.get_daemon_base_dir = exports.create_daemon_from_env = exports.DEFAULT_SESSION_NAME = exports.KEEPALIVE_INTERVAL = exports.DEFAULT_DAEMON_IDLE_TIMEOUT_MS = exports.BrowserDaemon = void 0;
+const fs_1 = __importDefault(require("fs"));
+const net_1 = __importDefault(require("net"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const playwright_core_1 = require("playwright-core");
+const connection_1 = require("./connection");
+const interaction_1 = require("./interaction");
+const ipc_1 = require("./ipc");
+const screenshot_1 = require("./screenshot");
+const snapshot_1 = require("./snapshot");
+const DEFAULT_SESSION_NAME = 'default';
+exports.DEFAULT_SESSION_NAME = DEFAULT_SESSION_NAME;
+const DEFAULT_DAEMON_IDLE_TIMEOUT_MS = 600_000;
+exports.DEFAULT_DAEMON_IDLE_TIMEOUT_MS = DEFAULT_DAEMON_IDLE_TIMEOUT_MS;
+const KEEPALIVE_INTERVAL = 30_000;
+exports.KEEPALIVE_INTERVAL = KEEPALIVE_INTERVAL;
+const MAX_TRACKED_REQUESTS = 200;
+const WINDOWS_PORT_BASE = 49_152;
+const WINDOWS_PORT_SPAN = 16_383;
+const is_object = (value) => {
+    return !!value && typeof value == 'object' && !Array.isArray(value);
+};
+const get_path_api = (platform) => platform == 'win32' ? path_1.default.win32 : path_1.default.posix;
+const format_error = (error) => {
+    if (error instanceof Error)
+        return error.message;
+    return String(error);
+};
+const normalize_session_name = (session_name) => {
+    const normalized = (session_name ?? DEFAULT_SESSION_NAME).trim();
+    if (!normalized)
+        throw new Error('Browser session name cannot be empty.');
+    if (!/^[A-Za-z0-9._-]+$/.test(normalized)) {
+        throw new Error('Browser session name may contain only letters, numbers, dots, '
+            + 'underscores, and hyphens.');
+    }
+    return normalized;
+};
+exports.normalize_session_name = normalize_session_name;
+const normalize_idle_timeout = (idle_timeout_ms) => {
+    if (idle_timeout_ms === undefined)
+        return DEFAULT_DAEMON_IDLE_TIMEOUT_MS;
+    if (!Number.isFinite(idle_timeout_ms) || idle_timeout_ms < 0) {
+        throw new Error('Browser idle timeout must be a non-negative number of '
+            + 'milliseconds.');
+    }
+    return Math.floor(idle_timeout_ms);
+};
+const parse_idle_timeout = (raw) => {
+    const normalized = raw?.trim();
+    if (!normalized)
+        return undefined;
+    return normalize_idle_timeout(Number(normalized));
+};
+exports.parse_idle_timeout = parse_idle_timeout;
+const get_daemon_base_dir = (opts = {}) => {
+    const platform = opts.platform ?? process.platform;
+    const env = opts.env ?? process.env;
+    const home_dir = opts.home_dir ?? os_1.default.homedir();
+    const path_api = get_path_api(platform);
+    const daemon_dir = opts.daemon_dir?.trim();
+    const env_daemon_dir = env['BRIGHTDATA_DAEMON_DIR']?.trim();
+    if (daemon_dir)
+        return daemon_dir;
+    if (env_daemon_dir)
+        return env_daemon_dir;
+    if (platform == 'darwin') {
+        return path_api.join(home_dir, 'Library', 'Application Support', 'brightdata-cli');
+    }
+    if (platform == 'win32')
+        return path_api.join(home_dir, 'AppData', 'Roaming', 'brightdata-cli');
+    const runtime_dir = env['XDG_RUNTIME_DIR']?.trim();
+    if (runtime_dir)
+        return path_api.join(runtime_dir, 'brightdata-cli');
+    return path_api.join(home_dir, '.brightdata-cli');
+};
+exports.get_daemon_base_dir = get_daemon_base_dir;
+const get_port_for_session = (session_name) => {
+    let hash = 0;
+    for (let i = 0; i < session_name.length; i++) {
+        hash = ((hash << 5) - hash + session_name.charCodeAt(i)) | 0;
+    }
+    return WINDOWS_PORT_BASE + ((hash >>> 0) % WINDOWS_PORT_SPAN);
+};
+exports.get_port_for_session = get_port_for_session;
+const get_daemon_transport = (session_name, opts = {}) => {
+    const normalized_session = normalize_session_name(session_name);
+    const platform = opts.platform ?? process.platform;
+    const base_dir = get_daemon_base_dir({ ...opts, platform });
+    const path_api = get_path_api(platform);
+    const pid_path = path_api.join(base_dir, `${normalized_session}.pid`);
+    if (platform == 'win32') {
+        const port = get_port_for_session(normalized_session);
+        return {
+            kind: 'tcp',
+            base_dir,
+            host: '127.0.0.1',
+            pid_path,
+            port,
+            port_path: path_api.join(base_dir, `${normalized_session}.port`),
+        };
+    }
+    return {
+        kind: 'unix',
+        base_dir,
+        pid_path,
+        socket_path: path_api.join(base_dir, `${normalized_session}.sock`),
+    };
+};
+exports.get_daemon_transport = get_daemon_transport;
+class BrowserDaemon {
+    state;
+    active_contexts = new WeakSet();
+    active_pages = new WeakSet();
+    deps;
+    idle_timeout_ms;
+    request_ids = new WeakMap();
+    transport;
+    active_requests = 0;
+    request_counter = 0;
+    server = null;
+    stop_promise = null;
+    constructor(opts, deps = {}) {
+        const session_name = normalize_session_name(opts.session_name);
+        const cdp_endpoint = opts.cdp_endpoint.trim();
+        if (!cdp_endpoint)
+            throw new Error('Browser daemon CDP endpoint cannot be empty.');
+        this.idle_timeout_ms = normalize_idle_timeout(opts.idle_timeout_ms);
+        this.transport = get_daemon_transport(session_name, {
+            daemon_dir: opts.daemon_dir,
+            env: opts.env,
+            home_dir: opts.home_dir,
+        });
+        this.state = {
+            browser: null,
+            page: null,
+            connected: false,
+            dom_refs: new Map(),
+            requests: new Map(),
+            cdp_endpoint,
+            session_name,
+            idle_timer: null,
+            keepalive_timer: null,
+        };
+        this.deps = {
+            clear_interval: clearInterval,
+            clear_timeout: clearTimeout,
+            connect_over_cdp: (endpoint) => playwright_core_1.chromium.connectOverCDP(endpoint),
+            create_server: net_1.default.createServer,
+            pid: () => process.pid,
+            set_interval: setInterval,
+            set_timeout: setTimeout,
+            ...deps,
+        };
+    }
+    get_transport() {
+        return this.transport;
+    }
+    is_running() {
+        return this.server?.listening === true;
+    }
+    async start() {
+        if (this.server?.listening)
+            return;
+        fs_1.default.mkdirSync(this.transport.base_dir, { recursive: true });
+        if (this.transport.kind == 'unix' &&
+            fs_1.default.existsSync(this.transport.socket_path)) {
+            fs_1.default.rmSync(this.transport.socket_path, { force: true });
+        }
+        const server = this.deps.create_server(socket => this.handle_socket(socket));
+        this.server = server;
+        try {
+            await new Promise((resolve, reject) => {
+                const on_error = (error) => {
+                    server.off('listening', on_listening);
+                    reject(error);
+                };
+                const on_listening = () => {
+                    server.off('error', on_error);
+                    resolve();
+                };
+                server.once('error', on_error);
+                server.once('listening', on_listening);
+                if (this.transport.kind == 'unix')
+                    server.listen(this.transport.socket_path);
+                else
+                    server.listen(this.transport.port, this.transport.host);
+            });
+            fs_1.default.writeFileSync(this.transport.pid_path, `${this.deps.pid()}`);
+            if (this.transport.kind == 'tcp') {
+                fs_1.default.writeFileSync(this.transport.port_path, `${this.transport.port}`);
+            }
+            this.schedule_idle_timer();
+            this.start_keepalive();
+        }
+        catch (error) {
+            this.server = null;
+            this.cleanup_transport_files();
+            try {
+                server.close();
+            }
+            catch (_error) { }
+            throw error;
+        }
+    }
+    async stop() {
+        if (this.stop_promise)
+            return this.stop_promise;
+        this.stop_promise = this.stop_internal();
+        try {
+            await this.stop_promise;
+        }
+        finally {
+            this.stop_promise = null;
+        }
+    }
+    async handle_request(request) {
+        this.begin_activity();
+        try {
+            const data = await this.execute_request(request);
+            return { id: request.id, success: true, data };
+        }
+        catch (error) {
+            return {
+                id: request.id,
+                success: false,
+                error: format_error(error),
+            };
+        }
+        finally {
+            this.end_activity();
+        }
+    }
+    begin_activity() {
+        this.active_requests++;
+        this.clear_idle_timer();
+    }
+    end_activity() {
+        this.active_requests = Math.max(0, this.active_requests - 1);
+        if (this.active_requests == 0 && !this.stop_promise)
+            this.schedule_idle_timer();
+    }
+    clear_idle_timer() {
+        if (this.state.idle_timer) {
+            this.deps.clear_timeout(this.state.idle_timer);
+            this.state.idle_timer = null;
+        }
+    }
+    schedule_idle_timer() {
+        this.clear_idle_timer();
+        if (this.idle_timeout_ms == 0)
+            return;
+        this.state.idle_timer = this.deps.set_timeout(() => {
+            void this.stop();
+        }, this.idle_timeout_ms);
+    }
+    clear_keepalive() {
+        if (this.state.keepalive_timer) {
+            this.deps.clear_interval(this.state.keepalive_timer);
+            this.state.keepalive_timer = null;
+        }
+    }
+    start_keepalive() {
+        this.clear_keepalive();
+        this.state.keepalive_timer = this.deps.set_interval(() => {
+            const browser = this.state.browser;
+            if (!browser)
+                return;
+            try {
+                browser.contexts();
+                this.state.connected = true;
+            }
+            catch (_error) {
+                if (this.state.browser == browser)
+                    (0, connection_1.clear_connection_state)(this.state);
+            }
+        }, KEEPALIVE_INTERVAL);
+    }
+    async stop_internal() {
+        this.clear_idle_timer();
+        this.clear_keepalive();
+        await this.close_browser();
+        const server = this.server;
+        this.server = null;
+        if (server) {
+            await new Promise(resolve => {
+                server.close(() => resolve());
+            });
+        }
+        this.cleanup_transport_files();
+    }
+    cleanup_transport_files() {
+        fs_1.default.rmSync(this.transport.pid_path, { force: true });
+        if (this.transport.kind == 'unix')
+            fs_1.default.rmSync(this.transport.socket_path, { force: true });
+        else
+            fs_1.default.rmSync(this.transport.port_path, { force: true });
+    }
+    handle_socket(socket) {
+        socket.setEncoding('utf8');
+        let buffer = '';
+        let queue = Promise.resolve();
+        socket.on('data', chunk => {
+            buffer += chunk;
+            let newline_index = buffer.indexOf('\n');
+            while (newline_index >= 0) {
+                const line = buffer.slice(0, newline_index).trim();
+                buffer = buffer.slice(newline_index + 1);
+                if (line) {
+                    queue = queue
+                        .then(() => this.process_line(socket, line))
+                        .catch(() => undefined);
+                }
+                newline_index = buffer.indexOf('\n');
+            }
+        });
+        socket.on('error', () => undefined);
+    }
+    async process_line(socket, line) {
+        let request = null;
+        let response;
+        try {
+            request = (0, ipc_1.parse_daemon_request)(JSON.parse(line));
+            response = await this.handle_request(request);
+        }
+        catch (error) {
+            response = {
+                id: request?.id ?? 'invalid',
+                success: false,
+                error: format_error(error),
+            };
+        }
+        if (!socket.destroyed)
+            socket.write(JSON.stringify(response) + '\n');
+        if (request?.action == 'close' && response.success) {
+            socket.end();
+            void this.stop();
+        }
+    }
+    async execute_request(request) {
+        switch (request.action) {
+            case 'back':
+                return this.handle_history_navigation('back');
+            case 'check':
+                return (0, interaction_1.handle_check)(await this.ensure_connected(), request.params, true);
+            case 'click':
+                return (0, interaction_1.handle_click)(await this.ensure_connected(), request.params);
+            case 'close':
+                await this.close_browser();
+                return { closed: true };
+            case 'cookies':
+                return this.handle_cookies();
+            case 'fill':
+                return (0, interaction_1.handle_fill)(await this.ensure_connected(), request.params);
+            case 'forward':
+                return this.handle_history_navigation('forward');
+            case 'get_html':
+                return (0, interaction_1.handle_get_html)(await this.ensure_connected(), request.params);
+            case 'get_text':
+                return (0, interaction_1.handle_get_text)(await this.ensure_connected(), request.params);
+            case 'hover':
+                return (0, interaction_1.handle_hover)(await this.ensure_connected(), request.params);
+            case 'navigate':
+                return this.handle_navigate(request.params);
+            case 'network':
+                return this.handle_network();
+            case 'ping':
+                return { alive: true, connected: this.state.connected };
+            case 'reload':
+                return this.handle_reload();
+            case 'screenshot':
+                return this.handle_screenshot(request.params);
+            case 'scroll':
+                return (0, interaction_1.handle_scroll)(await this.ensure_connected(), request.params);
+            case 'select':
+                return (0, interaction_1.handle_select)(await this.ensure_connected(), request.params);
+            case 'snapshot':
+                return this.handle_snapshot(request.params);
+            case 'status':
+                return this.handle_status();
+            case 'type':
+                return (0, interaction_1.handle_type)(await this.ensure_connected(), request.params);
+            case 'uncheck':
+                return (0, interaction_1.handle_check)(await this.ensure_connected(), request.params, false);
+            default:
+                throw new Error(`Unknown daemon action "${request.action}".`);
+        }
+    }
+    async handle_navigate(params) {
+        const url = params?.['url'];
+        if (typeof url != 'string' || !url.trim()) {
+            throw new Error('Navigate requires a non-empty "url" parameter.');
+        }
+        const new_endpoint = params?.['cdp_endpoint'];
+        if (new_endpoint !== undefined) {
+            if (typeof new_endpoint != 'string' || !new_endpoint.trim())
+                throw new Error('Navigate "cdp_endpoint" must be a non-empty string.');
+            if (new_endpoint.trim() !== this.state.cdp_endpoint)
+                await this.switch_cdp_endpoint(new_endpoint.trim());
+        }
+        const page = await this.ensure_connected();
+        this.state.dom_refs.clear();
+        this.state.requests.clear();
+        const response = await page.goto(url, { waitUntil: 'load' });
+        return this.create_navigation_result(page, response);
+    }
+    async switch_cdp_endpoint(new_endpoint) {
+        await this.close_browser();
+        this.state.cdp_endpoint = new_endpoint;
+        this.state.requests.clear();
+    }
+    async handle_history_navigation(direction) {
+        const page = await this.ensure_connected();
+        this.state.dom_refs.clear();
+        const response = direction == 'back'
+            ? await page.goBack({ waitUntil: 'load' })
+            : await page.goForward({ waitUntil: 'load' });
+        return this.create_navigation_result(page, response);
+    }
+    async handle_reload() {
+        const page = await this.ensure_connected();
+        this.state.dom_refs.clear();
+        const response = await page.reload({ waitUntil: 'load' });
+        return this.create_navigation_result(page, response);
+    }
+    async handle_cookies() {
+        const page = await this.ensure_connected();
+        return {
+            cookies: await page.context().cookies(),
+        };
+    }
+    async create_navigation_result(page, response) {
+        return {
+            status: response?.status() ?? null,
+            title: await page.title(),
+            url: page.url(),
+        };
+    }
+    async handle_snapshot(params) {
+        const compact = params?.['compact'];
+        const depth = params?.['depth'];
+        const interactive = params?.['interactive'];
+        const selector = params?.['selector'];
+        if (compact !== undefined && typeof compact != 'boolean') {
+            throw new Error('Snapshot "compact" parameter must be a boolean when provided.');
+        }
+        if (interactive !== undefined && typeof interactive != 'boolean') {
+            throw new Error('Snapshot "interactive" parameter must be a boolean when provided.');
+        }
+        if (depth !== undefined
+            && (typeof depth != 'number' || !Number.isInteger(depth) || depth < 0)) {
+            throw new Error('Snapshot "depth" parameter must be a non-negative integer when provided.');
+        }
+        if (selector !== undefined
+            && (typeof selector != 'string' || !selector.trim())) {
+            throw new Error('Snapshot "selector" parameter must be a non-empty string when provided.');
+        }
+        const wrap = params?.['wrap'];
+        if (wrap !== undefined && typeof wrap != 'boolean') {
+            throw new Error('Snapshot "wrap" parameter must be a boolean when provided.');
+        }
+        const page = await this.ensure_connected();
+        const result = await (0, snapshot_1.capture_snapshot)(page, {
+            compact: compact === true,
+            depth: depth,
+            interactive: interactive === true,
+            selector: typeof selector == 'string' ? selector.trim() : undefined,
+            wrap: wrap === true,
+        });
+        this.state.dom_refs.clear();
+        for (const entry of result.refs)
+            this.state.dom_refs.set(entry.ref, entry.selector);
+        return {
+            compact: result.compact,
+            depth: result.depth,
+            interactive: result.interactive,
+            ref_count: result.refs.length,
+            selector: result.selector,
+            snapshot: result.snapshot,
+            title: result.title,
+            url: result.url,
+            wrap: result.wrap,
+        };
+    }
+    async handle_screenshot(params) {
+        const base64 = params?.['base64'];
+        const full_page = params?.['full_page'];
+        const file_path = params?.['path'];
+        if (base64 !== undefined && typeof base64 != 'boolean') {
+            throw new Error('Screenshot "base64" parameter must be a boolean when provided.');
+        }
+        if (full_page !== undefined && typeof full_page != 'boolean') {
+            throw new Error('Screenshot "full_page" parameter must be a boolean when provided.');
+        }
+        if (file_path !== undefined
+            && (typeof file_path != 'string' || !file_path.trim())) {
+            throw new Error('Screenshot "path" parameter must be a non-empty string when provided.');
+        }
+        const page = await this.ensure_connected();
+        return await (0, screenshot_1.take_screenshot)(page, {
+            base64: base64 === true,
+            full_page: full_page === true,
+            path: typeof file_path == 'string' ? file_path.trim() : undefined,
+        });
+    }
+    handle_network() {
+        return {
+            requests: Array.from(this.state.requests.entries()).map(([id, tracked_request]) => ({ id, ...tracked_request })),
+        };
+    }
+    async handle_status() {
+        const page = this.state.page && !this.state.page.isClosed()
+            ? this.state.page
+            : null;
+        return {
+            connected: this.state.connected,
+            current_title: page ? await this.safe_page_title(page) : null,
+            current_url: page ? page.url() : null,
+            dom_refs: this.state.dom_refs.size,
+            idle_timeout_ms: this.idle_timeout_ms,
+            listener: this.transport.kind == 'unix'
+                ? {
+                    kind: 'unix',
+                    socket_path: this.transport.socket_path,
+                }
+                : {
+                    host: this.transport.host,
+                    kind: 'tcp',
+                    port: this.transport.port,
+                },
+            session_name: this.state.session_name,
+            tracked_requests: this.state.requests.size,
+        };
+    }
+    async safe_page_title(page) {
+        try {
+            return await page.title();
+        }
+        catch (_error) {
+            return null;
+        }
+    }
+    async ensure_connected() {
+        return (0, connection_1.ensure_connected)(this.state, {
+            connect_over_cdp: this.deps.connect_over_cdp,
+        }, {
+            on_context: context => this.track_context(context),
+            on_page: page => {
+                this.track_page(page);
+                this.state.page = page;
+            },
+        });
+    }
+    track_context(context) {
+        if (this.active_contexts.has(context))
+            return;
+        this.active_contexts.add(context);
+        context.on('page', page => {
+            this.track_page(page);
+            this.state.page = page;
+            this.state.dom_refs.clear();
+        });
+    }
+    track_page(page) {
+        if (this.active_pages.has(page))
+            return;
+        this.active_pages.add(page);
+        page.on('close', () => {
+            if (this.state.page == page) {
+                this.state.page = null;
+                this.state.dom_refs.clear();
+            }
+        });
+        page.on('request', request => this.track_request(request));
+        page.on('response', response => this.track_response(response));
+    }
+    track_request(request) {
+        const request_id = `r${++this.request_counter}`;
+        this.request_ids.set(request, request_id);
+        this.state.requests.set(request_id, {
+            method: request.method(),
+            url: request.url(),
+            resource_type: request.resourceType(),
+        });
+        this.trim_tracked_requests();
+    }
+    track_response(response) {
+        const request = response.request();
+        const existing_id = this.request_ids.get(request)
+            ?? `r${++this.request_counter}`;
+        if (!this.request_ids.has(request))
+            this.request_ids.set(request, existing_id);
+        const tracked_request = this.state.requests.get(existing_id) ?? {
+            method: request.method(),
+            url: request.url(),
+        };
+        tracked_request.status = response.status();
+        this.state.requests.set(existing_id, tracked_request);
+        this.trim_tracked_requests();
+    }
+    trim_tracked_requests() {
+        while (this.state.requests.size > MAX_TRACKED_REQUESTS) {
+            const oldest_id = this.state.requests.keys().next().value;
+            if (!oldest_id)
+                return;
+            this.state.requests.delete(oldest_id);
+        }
+    }
+    async close_browser() {
+        const browser = this.state.browser;
+        (0, connection_1.clear_connection_state)(this.state);
+        if (!browser)
+            return;
+        try {
+            await browser.close();
+        }
+        catch (_error) { }
+    }
+}
+exports.BrowserDaemon = BrowserDaemon;
+const create_daemon_from_env = (env = process.env) => {
+    const cdp_endpoint = env['BRIGHTDATA_CDP_ENDPOINT']?.trim();
+    if (!cdp_endpoint) {
+        throw new Error('BRIGHTDATA_CDP_ENDPOINT is required to start the browser daemon.');
+    }
+    return new BrowserDaemon({
+        cdp_endpoint,
+        idle_timeout_ms: parse_idle_timeout(env['BRIGHTDATA_IDLE_TIMEOUT_MS']),
+        session_name: env['BRIGHTDATA_SESSION'] ?? DEFAULT_SESSION_NAME,
+    });
+};
+exports.create_daemon_from_env = create_daemon_from_env;
+const start_daemon_from_env = async (env = process.env) => {
+    const daemon = create_daemon_from_env(env);
+    await daemon.start();
+    return daemon;
+};
+exports.start_daemon_from_env = start_daemon_from_env;
+//# sourceMappingURL=daemon.js.map