@blitzbrowser/blitzbrowser 1.0.0

package/src/components/browser-instance.process.ts

@@ -0,0 +1,193 @@
+import { ChildProcess, spawn } from 'child_process';
+import * as fsPromise from 'fs/promises';
+import { exit } from 'process';
+import puppeteer, { Browser } from 'puppeteer';
+
+let SIGTERM_RECEIVED = false;
+
+class BrowserInstanceProcess {
+
+    readonly #cdp_port = parseInt(process.env.CDP_PORT);
+    readonly #proxy_server_port = parseInt(process.env.PROXY_SERVER_PORT);
+    readonly #timezone = process.env.TZ;
+    readonly #user_data_folder = process.env.USER_DATA_FOLDER;
+
+    #xvfb_process: ChildProcess;
+    #puppeteer_process: Browser;
+
+    #is_closing = false;
+    #client_disconnected = false;
+
+    get launched() {
+        return this.#xvfb_process !== undefined && this.#puppeteer_process !== undefined;
+    }
+
+    async launch() {
+        if (SIGTERM_RECEIVED) {
+            console.log(`Can't launch, received sigterm before launching.`);
+            await this.close();
+            return;
+        }
+
+        console.log('Launching');
+
+        try {
+            const display_id = this.#cdp_port; // We use the port to guarantee only one instance has this screen
+            const display = `:${display_id}`;
+            const display_lock_file = `/tmp/.X${display_id}-lock`;
+
+            // When container is restarting, xvfb locks are not always released on file system.
+            await fsPromise.rm(display_lock_file, { force: true });
+
+            console.log('Creating xvfb process');
+            this.#xvfb_process = spawn('Xvfb', [display, '-screen', '0', '1920x1080x24'], { stdio: 'inherit' });
+
+            console.log('Creating puppeteer process');
+            this.#puppeteer_process = await puppeteer.launch({
+                headless: false,
+                executablePath: puppeteer.executablePath(),
+                userDataDir: this.#user_data_folder,
+                dumpio: true,
+                handleSIGTERM: false,
+                args: [
+                    `--remote-debugging-port=${this.#cdp_port}`,
+                    '--remote-debugging-address=0.0.0.0',
+                    '--no-first-run',
+                    '--no-default-browser-check',
+                    `--display=${display}`,
+                    '--disable-breakpad', // Disables crash reporting
+                    '--disable-component-update', // Disables updates to internal components
+                    '--disable-print-preview', // Disables print preview
+                    '--disable-domain-reliability', // Disables domain reliability service(client-side reliability monitoring system for Google sites)
+                    '--disk-cache-dir=/dev/null', // Prevents disk caching
+                    '--no-pings', // Disables hyperlink auditing pings
+                    '--disable-notifications', // Prevents web push notifications
+                    '--disable-features=TranslateUI', // Disables the translate UI
+                    '--disable-background-networking',
+                    '--disable-sync',
+                    '--disable-extensions',
+                    '--disable-signin-promo',
+                    '--disable-gpu',
+                    '--metrics-recording-only', // Record metrics, but doesn't report them
+                    '--disable-features=PersistentHistograms', // Prevents generation of BrowserMetrics files on disk
+                    `--start-maximized`,
+                    `--proxy-server=http://127.0.0.1:${this.#proxy_server_port}`,
+                    process.env.DISABLE_SHM === 'true' ? '--disable-dev-shm-usage' : undefined
+                ].filter(s => typeof s === 'string'),
+                env: {
+                    TZ: this.#timezone
+                },
+            });
+            console.log('Created puppeteer process.');
+
+            this.#puppeteer_process.on('disconnected', () => {
+                this.#client_disconnected = true;
+                console.log('Client disconnected');
+            });
+
+            const interval_id = setInterval(() => {
+                if (this.#puppeteer_process.connected) {
+                    return;
+                }
+
+                clearInterval(interval_id);
+
+                if (this.#is_closing) {
+                    return;
+                }
+
+                console.log(`Puppeteer not connected.`);
+
+                this.close();
+            }, 200);
+        } catch (e) {
+            console.log('Error while launching processes', e);
+            this.close();
+        } finally {
+            if (SIGTERM_RECEIVED) {
+                this.close();
+                return;
+            }
+        }
+    }
+
+    async close() {
+        if (this.#is_closing) {
+            return;
+        }
+
+        this.#is_closing = true;
+
+        console.log('Closing.');
+
+        // We need to close puppeteer first. All other processes HAVE to be closed after. Otherwise it crashes puppeteer.
+        if (this.#puppeteer_process) {
+            try {
+                // If the client disconnected, it means puppeteer is already closing the browser process.
+                // Calling close() if already closing corrupt the user data. I lost 5 days over this concurrent closing issue.
+                if (!this.#client_disconnected && this.#puppeteer_process.connected) {
+                    await this.#puppeteer_process.close();
+                }
+
+                await this.#waitGoogleChromeKilled();
+            } catch (e) {
+                console.error('Error while killing puppeteer', e?.stack);
+            }
+        }
+
+        if (this.#xvfb_process) {
+            try {
+                this.#xvfb_process.kill();
+            } catch (e) {
+                console.error('Error while killing xvfb process', e?.stack);
+            }
+        }
+
+        exit(0);
+    }
+
+    async #waitGoogleChromeKilled() {
+        let count = 0;
+
+        await new Promise((res) => {
+            const interval_id = setInterval(() => {
+                if (count === 60) { // 3 seconds
+                    clearInterval(interval_id);
+                    res(undefined);
+                    return;
+                }
+
+                const ps = spawn('ps', ['ax']);
+
+                ps.on('exit', () => {
+                    count++;
+                });
+
+                ps.stdout.on('data', (data: Buffer) => {
+                    const message = data.toString();
+                    if (!message.includes(this.#user_data_folder)) {
+                        clearInterval(interval_id);
+                        res(undefined);
+                    }
+                })
+            }, 50);
+        });
+    }
+}
+
+let browser_instance_process: BrowserInstanceProcess;
+
+process.on('SIGTERM', () => {
+    SIGTERM_RECEIVED = true;
+    console.log('Received SIGTERM');
+
+    // If we receive sigterm while launching processes, we are going to wait for processes to finish launching.
+    // It guarantees that google chrome fully launched before killing it.
+    if (browser_instance_process && browser_instance_process.launched) {
+        browser_instance_process.close();
+    }
+});
+
+browser_instance_process = new BrowserInstanceProcess();
+
+browser_instance_process.launch();

package/src/controllers/cdp.controller.ts

@@ -0,0 +1,122 @@
+import { Controller, Logger, OnModuleInit } from '@nestjs/common';
+import { HttpAdapterHost } from '@nestjs/core';
+import { ExpressAdapter } from '@nestjs/platform-express';
+import { BrowserInstance, ConnectionOptionsEvent } from 'src/components/browser-instance.component';
+import { BrowserPoolService } from 'src/services/browser-pool.service';
+import { WebSocketServer } from 'ws';
+import z from 'zod';
+import { Message, Tunnel } from '@blitzbrowser/tunnel';
+
+export const PROXY_URL_QUERY_PARAM = 'proxyUrl';
+export const TIMEZONE_QUERY_PARAM = 'timezone';
+export const USER_DATA_ID_QUERY_PARAM = 'userDataId';
+export const USER_DATA_READ_ONLY_QUERY_PARAM = 'userDataReadOnly';
+
+const ConnectionOptionQueryParams = z.object({
+  proxy_url: z.url().optional(),
+  timezone: z.string().optional(),
+  user_data_id: z.string().optional(),
+  user_data_read_only: z.boolean().optional().default(false),
+});
+
+type ConnectionOptionQueryParams = z.infer<typeof ConnectionOptionQueryParams>;
+
+@Controller()
+export class CDPController implements OnModuleInit {
+
+  static readonly INTERNAL_SERVER_ERROR_CODE = 4000;
+  static readonly BAD_REQUEST_CODE = 4002;
+
+  readonly #logger = new Logger(CDPController.name);
+
+  constructor(
+    private readonly browser_pool_service: BrowserPoolService,
+    private readonly http_adapter_host: HttpAdapterHost,
+  ) { }
+
+  async onModuleInit() {
+    const websocket_server = new WebSocketServer({ server: (this.http_adapter_host.httpAdapter as ExpressAdapter).getHttpServer() });
+
+    websocket_server.on('connection', (cdp_websocket_client, req) => {
+      try {
+        const url = new URL(`http://127.0.0.1${req.url}`);
+        const parsed_connection_options = this.#parseConnectionOptionQueryParams(url);
+
+        if (!parsed_connection_options.success) {
+          cdp_websocket_client.close(CDPController.BAD_REQUEST_CODE, parsed_connection_options.error.message.trim());
+          return;
+        }
+
+        const browser_instance = this.browser_pool_service.createBrowserInstance();
+
+        const tunnel = new Tunnel();
+
+        tunnel.on('message', message => {
+          if (message.channel_id === BrowserInstance.CDP_CHANNEL_ID) {
+            cdp_websocket_client.send(message.data.toString('utf8'), { binary: false });
+          }
+        });
+
+        tunnel.once('closed', () => {
+          cdp_websocket_client.close();
+        })
+
+        cdp_websocket_client.on('message', message => {
+          tunnel.receiveMessage(Message.of(BrowserInstance.CDP_CHANNEL_ID, message.toString('utf8')));
+        });
+
+        const ping_interval_id = setInterval(() => {
+          cdp_websocket_client.ping();
+        }, 3000);
+
+        cdp_websocket_client.on('close', () => {
+          clearInterval(ping_interval_id);
+          tunnel.close();
+        });
+
+        cdp_websocket_client.on('error', err => {
+          this.#logger.error(`Error with cdp websocket.`, err?.stack || err);
+        });
+
+        browser_instance.connectTunnel(tunnel);
+
+        tunnel.receiveMessage(Message.of(BrowserInstance.EVENT_CHANNEL_ID, JSON.stringify({
+          type: 'CONNECTION_OPTIONS',
+          options: {
+            ...parsed_connection_options.data
+          }
+        } satisfies ConnectionOptionsEvent)));
+
+        this.#logger.log('Sent connection options');
+      } catch (e) {
+        this.#logger.error(`Error while handling client websocket.`, e?.stack || e)
+
+        cdp_websocket_client.close(CDPController.INTERNAL_SERVER_ERROR_CODE, e?.stack || e);
+        return;
+      }
+    });
+
+    websocket_server.once('close', () => {
+      this.browser_pool_service.shutdown();
+    });
+
+    websocket_server.once('error', (err) => {
+      this.#logger.error('Error with server websocket.', err?.stack || err);
+      this.browser_pool_service.shutdown();
+    });
+  }
+
+  #parseConnectionOptionQueryParams(url: URL) {
+    return ConnectionOptionQueryParams.safeParse({
+      timezone: this.getQueryParamValue(TIMEZONE_QUERY_PARAM, url),
+      proxy_url: this.getQueryParamValue(PROXY_URL_QUERY_PARAM, url),
+      user_data_id: this.getQueryParamValue(USER_DATA_ID_QUERY_PARAM, url),
+      user_data_read_only: this.getQueryParamValue(USER_DATA_READ_ONLY_QUERY_PARAM, url)?.toLowerCase() === 'true',
+    });
+  }
+
+  private getQueryParamValue(param: string, url: URL) {
+    return url.searchParams.has(param) ? url.searchParams.get(param) : undefined;
+  }
+
+}

package/src/controllers/pool-status.controller.ts

@@ -0,0 +1,18 @@
+import { Controller, Get } from '@nestjs/common';
+import { BrowserPoolService } from 'src/services/browser-pool.service';
+
+@Controller('/status')
+export class PoolStatusController {
+
+  constructor(private readonly pool_service: BrowserPoolService) { }
+
+  @Get()
+  getStatus() {
+    return {
+      sigterm_received: this.pool_service.sigterm_received,
+      browsers_running: this.pool_service.nb_browser_instances_alive,
+      max_browsers: this.pool_service.max_browser_instances,
+    };
+  }
+
+}

package/src/main.ts

@@ -0,0 +1,9 @@
+import { NestFactory } from '@nestjs/core';
+import { AppModule } from './app.module';
+
+async function bootstrap() {
+  const app = await NestFactory.create(AppModule);
+  await app.listen(process.env.PORT || 9999);
+}
+
+bootstrap();

package/src/services/browser-pool.service.ts

@@ -0,0 +1,139 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { BrowserInstance } from 'src/components/browser-instance.component';
+import * as EventEmitter from 'events';
+import { ModuleRef } from '@nestjs/core';
+
+type PoolServiceEvents = {
+  browser_instance_created: [BrowserInstance];
+}
+
+export interface BrowserPoolStatus {
+  id: string;
+  started_at: string;
+  max_browser_instances: number;
+  tags: { [key: string]: string; };
+}
+
+@Injectable()
+export class BrowserPoolService extends EventEmitter<PoolServiceEvents> implements OnModuleInit {
+
+  private readonly logger = new Logger(BrowserPoolService.name);
+
+  readonly #id: string = crypto.randomUUID();
+  readonly #started_at = new Date().toISOString();
+  readonly #tags: { [key: string]: string; } = {};
+
+  readonly max_browser_instances = parseInt(process.env.MAX_BROWSER_INSTANCES);
+
+  readonly #browser_instances = new Map<string, BrowserInstance>();
+
+  #sigterm_received: boolean = false;
+
+  constructor(
+    private readonly module_ref: ModuleRef,
+  ) {
+    super();
+
+    (process.env.TAGS || '').split(',').forEach(tag => {
+      const [key, value] = tag.split('=');
+
+      this.#tags[key] = value;
+    })
+  }
+
+  get status(): BrowserPoolStatus {
+    return {
+      id: this.#id,
+      started_at: this.#started_at,
+      max_browser_instances: this.max_browser_instances,
+      tags: this.#tags
+    };
+  }
+
+  get id() {
+    return this.#id;
+  }
+
+  get started_at() {
+    return this.#started_at;
+  }
+
+  get tags() {
+    return this.#tags;
+  }
+
+  get sigterm_received() {
+    return this.#sigterm_received;
+  }
+
+  get nb_browser_instances_alive() {
+    return this.#browser_instances.size;
+  }
+
+  get browser_instances() {
+    return [...this.#browser_instances.values()]
+  }
+
+  async onModuleInit() {
+    process.on('SIGTERM', () => {
+      this.logger.log('SIGTERM received');
+      this.shutdown();
+    });
+  }
+
+  getBrowserInstanceById(id: string): BrowserInstance | undefined {
+    return this.#browser_instances.get(id);
+  }
+
+  createBrowserInstance(id: string = crypto.randomUUID()) {
+    if (this.sigterm_received) {
+      this.logger.log(`Can't create new browser instance. Sigterm has been received.`);
+      return;
+    }
+
+    const browser_instance = new BrowserInstance(id, this.module_ref);
+
+    this.logger.log(`Created browser instance ${browser_instance.id}.`);
+
+    this.emit('browser_instance_created', browser_instance);
+
+    browser_instance.on('terminated', () => {
+      this.logger.log(`Browser instance ${browser_instance.id} terminated. Removing from pool.`);
+      this.#browser_instances.delete(browser_instance.id);
+    });
+
+    this.#browser_instances.set(browser_instance.id, browser_instance);
+
+    return browser_instance;
+  }
+
+  async shutdown() {
+    if (this.#sigterm_received) {
+      return;
+    }
+
+    this.#sigterm_received = true;
+
+    this.logger.log('Shutdown requested.');
+
+    for (const browser_instance of this.#browser_instances.values()) {
+      if (browser_instance.in_use) {
+        continue;
+      }
+
+      await browser_instance.close();
+    }
+
+    setInterval(async () => {
+      if (this.#browser_instances.size !== 0) {
+        this.logger.log(`Waiting for ${this.#browser_instances.size} browser instance(s) to close.`);
+        return;
+      }
+
+      this.logger.log(`All browser instances are closed.`);
+
+      process.exit(0);
+    }, 200);
+  }
+
+}

package/src/services/timezone.service.ts

@@ -0,0 +1,42 @@
+import { Injectable, Logger } from "@nestjs/common";
+import axios from "axios";
+
+@Injectable()
+export class TimezoneService {
+
+    private readonly logger = new Logger(TimezoneService.name);
+
+    #default_timezone: Promise<string>;
+
+    getDefaultTimezone() {
+        if (!this.#default_timezone) {
+            this.#default_timezone = new Promise(async (res) => {
+                res((await (await fetch('http://ip-api.com/json?fields=timezone')).json()).timezone);
+            });
+        }
+
+        return this.#default_timezone;
+    }
+
+    async getProxyTimezone(proxy_url: string): Promise<string> {
+        try {
+            const url = new URL(proxy_url);
+            const response = await axios.get('http://ip-api.com/json?fields=timezone', {
+                proxy: {
+                    protocol: url.protocol,
+                    host: url.hostname,
+                    port: parseInt(url.port),
+                    auth: {
+                        username: url.username,
+                        password: url.password,
+                    }
+                }
+            });
+
+            return response.data.timezone;
+        } catch (e) {
+            this.logger.error(`Error while getting timezone.`, e);
+        }
+    }
+
+}

package/src/services/user-data.service.ts

@@ -0,0 +1,111 @@
+import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { Injectable, Logger } from "@nestjs/common";
+import * as fs from 'fs';
+import * as fsPromise from 'fs/promises';
+import * as zlib from 'zlib';
+import * as tar from 'tar';
+import LimitStream from "src/transforms/limit-stream";
+
+export abstract class UserDataService {
+
+    abstract downloadUserData(id: string, user_data_folder: string): Promise<void>;
+    abstract uploadUserData(id: string, user_data_folder: string): Promise<void>;
+
+    /**
+     * 
+     * @param user_data_folder The user data folder to tar
+     * @returns The tar file path
+     */
+    protected async tarUserDataFolder(user_data_folder: string): Promise<string> {
+        const tar_file = `${crypto.randomUUID()}.tar.gz`;
+
+        await this.#cleanUserDataFolder(user_data_folder);
+
+        await tar.create({
+            gzip: true,
+            file: tar_file,
+            cwd: user_data_folder
+        }, ['.']);
+
+        return tar_file;
+    }
+
+    protected async untarUserDataFolder(tar_file: string, user_data_folder: string) {
+        await new Promise((res, rej) => {
+            fs.createReadStream(tar_file)
+                .pipe(zlib.createGunzip())
+                .pipe(new LimitStream(104857600)) // Max 100 MB decompression
+                .pipe(tar.extract({ cwd: user_data_folder }))
+                .on('finish', () => {
+                    res(undefined);
+                })
+                .on('error', (err) => {
+                    rej(err);
+                });
+        });
+        await this.#cleanUserDataFolder(user_data_folder);
+    }
+
+    /**
+     * Remove the undesired files(locks, logs, metrics) and folders from the user data
+     * @param user_data_folder 
+     */
+    async #cleanUserDataFolder(user_data_folder: string) {
+        await Promise.allSettled([
+            fsPromise.rm(`${user_data_folder}/Default/Sessions`, { recursive: true, force: true }),
+            fsPromise.rm(`${user_data_folder}/SingletonLock`, { force: true }),
+            fsPromise.rm(`${user_data_folder}/SingletonCookie`, { force: true }),
+            fsPromise.rm(`${user_data_folder}/SingletonSocket`, { force: true }),
+        ]);
+    }
+
+}
+
+@Injectable()
+export class UserDataS3Service extends UserDataService {
+
+    private readonly logger = new Logger(UserDataS3Service.name);
+
+    constructor(
+        private readonly s3_client: S3Client,
+    ) {
+        super();
+    }
+
+    async downloadUserData(id: string, user_data_folder: string) {
+        try {
+            const tar_file = `/tmp/${crypto.randomUUID()}`;
+            const response = await this.s3_client.send(new GetObjectCommand({
+                Bucket: process.env.S3_USER_DATA_BUCKET,
+                Key: id,
+            }));
+
+            await fsPromise.writeFile(tar_file, await response.Body.transformToByteArray());
+            await this.untarUserDataFolder(tar_file, user_data_folder);
+            await fsPromise.rm(tar_file);
+
+            this.logger.log(`Downloaded user data ${id} folder:${user_data_folder}`);
+        } catch (e) {
+            if (e.Code === 'NoSuchKey') {
+                return;
+            }
+
+            throw e;
+        }
+    }
+
+    async uploadUserData(id: string, user_data_folder: string) {
+        const tar_file = await this.tarUserDataFolder(user_data_folder);
+
+        await this.s3_client.send(new PutObjectCommand({
+            Bucket: process.env.S3_USER_DATA_BUCKET,
+            Key: id,
+            Body: await fsPromise.readFile(tar_file)
+        }));
+
+        await fsPromise.rm(tar_file);
+
+        this.logger.log(`Uploaded user data ${id} folder:${user_data_folder}`);
+    }
+
+}

package/src/transforms/limit-stream.ts

@@ -0,0 +1,30 @@
+import { Transform, TransformCallback, TransformOptions } from 'stream';
+
+/**
+ * A Transform stream that enforces a maximum size limit on the data passing through.
+ * @param {number} limitInBytes - The maximum allowed uncompressed size.
+ */
+export default class LimitStream extends Transform {
+
+    private bytes_processed = 0;
+
+    constructor(
+        private readonly byte_limit: number,
+        options?: TransformOptions
+    ) {
+        super(options);
+    }
+
+    _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback) {
+        this.bytes_processed += chunk.length;
+
+        if (this.bytes_processed > this.byte_limit) {
+            const error = new Error(`Decompression limit exceeded. Max size: ${this.byte_limit} bytes`);
+            this.destroy(error);
+            return callback(error);
+        }
+
+        this.push(chunk);
+        callback();
+    }
+}

package/tsconfig.build.json

@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+}

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "declaration": true,
+    "removeComments": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "ES2021",
+    "sourceMap": true,
+    "outDir": "./dist",
+    "baseUrl": "./",
+    "incremental": true,
+    "skipLibCheck": true,
+    "strictNullChecks": false,
+    "noImplicitAny": false,
+    "strictBindCallApply": false,
+    "forceConsistentCasingInFileNames": false,
+    "noFallthroughCasesInSwitch": false
+  }
+}