@mefistofelix/cdp.js 0.1.2 → 0.3.0

package/cdp.js

@@ -2,6 +2,7 @@ import * as child_process from 'node:child_process'
 import * as fs from 'node:fs/promises'
 import { existsSync } from 'node:fs'
 import * as path from 'node:path'
+import * as crypto from 'node:crypto'
 import { once, setMaxListeners } from 'node:events'
 
 export class util {
@@ -45,15 +46,19 @@ export class util {
 
 export class jsrpc extends WebSocket {
   constructor(url, options = {}) {
-    super(url)
-    setMaxListeners(0, this)
-    this.id = 0
-    this.pending = {}
-    this.timeout_ms = options.request_timeout_ms ?? 30000
-    this.addEventListener('message', event => {
+    const socket = typeof url === 'string' || url instanceof URL ? super(url) : url
+    setMaxListeners(0, socket)
+    socket.id = 0
+    socket.pending = {}
+    socket.timeout_ms = options.request_timeout_ms ?? 30000
+    socket.req = jsrpc.prototype.req
+    socket.notify = jsrpc.prototype.notify
+    socket.addEventListener('message', event => {
+      if (!event.data) return
       const message = JSON.parse(event.data)
-      util.emit(this, message.id === undefined ? 'notify' : 'rpc_' + message.id, message)
+      util.emit(socket, message.id === undefined ? 'notify' : 'rpc_' + message.id, message)
     })
+    return socket
   }
 
   async req(request, timeout_ms = this.timeout_ms) {
@@ -91,6 +96,12 @@ export class jsrpc extends WebSocket {
 }
 
 export class browser {
+  static extension_id(directory) {
+    const normalized = path.resolve(directory).replace(/^[a-z]:/, drive => drive.toUpperCase())
+    return crypto.createHash('sha256').update(normalized, process.platform === 'win32' ? 'utf16le' : 'utf8')
+      .digest('hex').slice(0, 32).replace(/./g, digit => 'abcdefghijklmnop'[parseInt(digit, 16)])
+  }
+
   static find_executable_path() {
     if (process.env.CDP_BROWSER) return process.env.CDP_BROWSER
     const candidates = process.platform === 'win32'
@@ -167,11 +178,23 @@ export class browser {
       args.headless = 'new'
       args['window-size'] = '1440,900'
     }
-    if (options.extensions !== true) {
+    if (Array.isArray(options.extensions)) args['disable-extensions-except'] = options.extensions
+    if (options.extensions !== true && !Array.isArray(options.extensions)) {
       args['disable-extensions'] = true
       args['disable-component-extensions-with-background-pages'] = true
     }
+    if (options.cdp_ext) {
+      args['remote-debugging-port'] = false
+      delete args['disable-extensions']
+      delete args['disable-component-extensions-with-background-pages']
+      if (options.extension_path) args['disable-extensions-except'] = [
+        options.extension_path,
+        ...(args['disable-extensions-except'] ?? (options.extensions === true ? ['*'] : [])),
+      ]
+    }
     if (options.images === false) args['blink-settings'] = 'imagesEnabled=false'
+    if (options.proxy === false) args['no-proxy-server'] = true
+    else if (options.proxy != null) args['proxy-server'] = options.proxy
     if (options.translations !== true) {
       args['disable-features'].push('Translate', 'TranslateToast', 'EnableTranslatePdf')
     }
@@ -180,7 +203,39 @@ export class browser {
       args['disable-sync'] = true
       args['disable-features'].push('DiceWebSigninInterception', 'SigninPromoOnAvatarPill')
     }
-    return Object.assign(args, options.args)
+    Object.assign(args, options.args)
+    const extensions = args['disable-extensions-except']
+    if (extensions != null && extensions !== false) {
+      if (extensions === true) args['disable-extensions-except'] = []
+      else args['disable-extensions-except'] = Array.isArray(extensions) ? [...extensions] : String(extensions).split(',')
+      const features = args['disable-features']
+      args['disable-features'] = Array.isArray(features) ? [...features] : features ? String(features).split(',') : []
+      const feature = 'DisableDisableExtensionsExceptCommandLineSwitch'
+      if (!args['disable-features'].includes(feature)) args['disable-features'].push(feature)
+    }
+    return args
+  }
+
+  static async profile_extension_paths(args) {
+    if (!args['user-data-dir']) throw new Error('Reading profile extensions requires user-data-dir')
+    const profile = path.join(args['user-data-dir'], args['profile-directory'] || 'Default')
+    const settings = {}
+    for (const filename of ['Preferences', 'Secure Preferences']) {
+      const content = await fs.readFile(path.join(profile, filename), 'utf8').catch(error => {
+        if (error.code !== 'ENOENT') throw error
+        return '{}'
+      })
+      Object.assign(settings, JSON.parse(content).extensions?.settings)
+    }
+    for (const [id, extension] of Object.entries(settings)) {
+      if (!extension.path || [5, 10].includes(extension.location)) delete settings[id]
+      else settings[id] = {
+        path: path.isAbsolute(extension.path) ? extension.path : path.join(profile, 'Extensions', extension.path),
+        enabled: extension.state !== 0 &&
+          !(Array.isArray(extension.disable_reasons) ? extension.disable_reasons.length : extension.disable_reasons),
+      }
+    }
+    return settings
   }
 
   static async update_profile_preferences(filename, preferences) {
@@ -201,19 +256,20 @@ export class browser {
 
   static async launch(options) {
     const args = browser.build_args(options)
-    const filename = path.join(args['user-data-dir'], args['profile-directory'] || 'Default', 'Preferences')
-    await browser.update_profile_preferences(filename, {
-      'translate.enabled': options.translations === true,
-      'signin.allowed': options.login === true,
-      'signin.allowed_on_next_startup': options.login === true,
-      ...options.preferences,
-    })
+    const extensions = args['disable-extensions-except']
+    if (extensions && extensions.includes('*')) {
+      const installed = Object.values(await browser.profile_extension_paths(args))
+        .filter(extension => extension.enabled).map(extension => extension.path)
+      args['disable-extensions-except'] = [...new Set(extensions.flatMap(directory =>
+        directory === '*' ? installed : directory,
+      ))]
+    }
     const proc = child_process.spawn(options.executable_path || browser.find_executable_path(), [...util.args_to_strings(args), 'about:blank'], {
       windowsHide: options.windowsHide ?? (args.headless != null && args.headless !== false),
       stdio: ['ignore', 'ignore', 'pipe'],
     })
     let diagnostic = ''
-    const ready = util.on_first(proc, 'ready exit', options.connect_timeout_ms ?? 15000)
+    const ready = util.on_first(proc, options.cdp_ext ? 'spawn exit' : 'ready exit', options.connect_timeout_ms ?? 15000)
     const data = chunk => {
       diagnostic = (diagnostic + chunk).slice(-8192)
       const match = diagnostic.match(/DevTools listening on (ws:\/\/\S+)/)
@@ -285,14 +341,112 @@ const custom_methods = {
 }
 
 export class cdp extends EventTarget {
-  constructor({ base_path = '.cdp', ...options } = {}) {
+  constructor({ base_path = '.cdp', reverse_server = {}, ...options } = {}) {
     super()
     this.base_path = path.resolve(base_path)
     this.options = options
+    this.reverse_server = { ...reverse_server }
     this.browsers = Object.create(null)
+    this.reverse_sockets = Object.create(null)
     this.custom_methods = { ...custom_methods }
   }
 
+  listen_reverse({
+    hostname = '127.0.0.1',
+    port = 9223,
+  } = this.reverse_server) {
+    if (!globalThis.Deno && !globalThis.Bun) throw new Error('The reverse WebSocket server requires Deno or Bun')
+    if (this.server) throw new Error('Reverse server is already listening')
+    const accept = (socket, origin) => {
+      socket.addEventListener('message', event => {
+        const id = JSON.parse(event.data).browser
+        if (id !== origin.slice('chrome-extension://'.length) ||
+            this.reverse_sockets[id]?.readyState < WebSocket.CLOSING) {
+          return socket.close(1008, 'Invalid or already connected extension ID')
+        }
+        this.reverse_sockets[id] = socket
+        socket.addEventListener('close', () => {
+          if (this.reverse_sockets[id] === socket) delete this.reverse_sockets[id]
+          this._close_reverse_if_idle()
+        })
+        util.emit(this, 'reverse_' + encodeURIComponent(id), socket)
+      }, { once: true })
+    }
+    const fetch = (request, server) => {
+      const origin = request.headers.get('origin')
+      if (request.headers.get('upgrade') !== 'websocket' ||
+          !origin?.startsWith('chrome-extension://')) {
+        return new Response(null, { status: 403 })
+      }
+      if (globalThis.Bun) {
+        if (server.upgrade(request, {
+          data: {
+            origin,
+            socket: new EventTarget(),
+          },
+        })) return
+        return new Response(null, { status: 400 })
+      }
+      const { socket, response } = Deno.upgradeWebSocket(request)
+      accept(socket, origin)
+      return response
+    }
+    if (globalThis.Bun) {
+      const server = this.server = Bun.serve({
+        hostname,
+        port,
+        fetch,
+        websocket: {
+          open(socket) {
+            const events = socket.data.socket
+            events.send = socket.send.bind(socket)
+            events.close = socket.close.bind(socket)
+            Object.defineProperty(events, 'readyState', { get: () => socket.readyState })
+            accept(events, socket.data.origin)
+          },
+          message(socket, data) {
+            socket.data.socket.dispatchEvent(new MessageEvent('message', { data }))
+          },
+          close(socket, code, reason) {
+            socket.data.socket.dispatchEvent(Object.assign(new Event('close'), {
+              code,
+              reason,
+            }))
+          },
+        },
+      })
+      server.addr = {
+        hostname,
+        port: server.port,
+      }
+      const finished = Promise.withResolvers()
+      server.finished = finished.promise
+      server.shutdown = () => server.stop().then(finished.resolve)
+    } else this.server = Deno.serve({
+      hostname,
+      port,
+    }, fetch)
+    this.reverse_server = {
+      hostname,
+      port: this.server.addr.port,
+    }
+    return this.server
+  }
+
+  _close_reverse_if_idle() {
+    if (this.server_closing) return this.server_closing
+    if (!this.server || Object.values(this.reverse_sockets).some(socket => socket.readyState < WebSocket.CLOSING)) return
+    for (const record of Object.values(this.browsers)) {
+      if (!record.reverse && !record.cdp_ext) continue
+      if (record.connecting) return
+      if (record.proc?.exitCode === null && record.proc.signalCode === null) return
+    }
+    return this.server_closing = this.server.shutdown().finally(() => {
+      this.server = null
+      this.server_closing = null
+    })
+  }
+
   evaluate_xpath(params, fn, options) {
     if (params.target == null) throw new Error('Custom method requires target')
     const xpath = util.normalize_xpath(params.xpath)
@@ -326,7 +480,8 @@ export class cdp extends EventTarget {
       browser: browser.name,
     }
     const sessionId = params.sessionId || message.sessionId
-    const targetId = params.targetInfo?.targetId || params.targetId || browser.session_targets[sessionId]
+    const targetId = params.targetInfo?.targetId || params.targetId || browser.session_targets[sessionId] ||
+      (params.tabId >= 0 ? String(params.tabId) : undefined)
     const target = Object.values(browser.targets).find(value => value.targetId === targetId)
     if (params.targetInfo) {
       Object.assign(browser.target_info[targetId] ??= {}, params.targetInfo)
@@ -347,26 +502,152 @@ export class cdp extends EventTarget {
     if (['Target.targetDestroyed', 'Target.targetCrashed'].includes(message.method)) {
       this._detach(browser, info?.sessionId)
       delete browser.target_info[targetId]
+      this._close_contexts_if_idle(browser)
+    }
+  }
+
+  async _context(browser, spec) {
+    if (typeof spec === 'string') spec = { name: spec }
+    const context = browser.contexts[spec.name] ??= {
+      ...spec,
+      browserContextId: null,
+    }
+    await context.closing
+    if (!context.browserContextId) {
+      context.connecting ??= browser.socket.req({
+        method: 'Target.createBrowserContext',
+        params: {
+          proxyServer: context.proxy === false ? 'direct://' : context.proxy,
+          ...context.create_params,
+        },
+      }).then(result => {
+        context.browserContextId = result.browserContextId
+      }).finally(() => {
+        context.connecting = null
+      })
+      await context.connecting
+    }
+    return context
+  }
+
+  _dispose_context(browser, context, params = { browserContextId: context.browserContextId }) {
+    const id = context.browserContextId
+    return context.closing ??= browser.socket.req({
+      method: 'Target.disposeBrowserContext',
+      params,
+    }).then(result => {
+      context.browserContextId = null
+      for (const [targetId, info] of Object.entries(browser.target_info)) {
+        if (info.browserContextId !== id) continue
+        this._detach(browser, info.sessionId)
+        delete browser.target_info[targetId]
+      }
+      util.emit(this, 'context_closed', {
+        browser: browser.name,
+        context: context.name,
+        browserContextId: id,
+      })
+      return result
+    }).finally(() => {
+      context.closing = null
+    })
+  }
+
+  _close_contexts_if_idle(browser) {
+    if (browser.context_checking) {
+      browser.context_check_pending = true
+      return browser.context_checking
     }
+    if (browser.connecting || browser.closing || browser.socket?.readyState !== WebSocket.OPEN) return
+    const contexts = Object.values(browser.contexts).filter(context =>
+      context.browserContextId && !context.connecting && !context.closing &&
+      !Object.values(browser.target_info).some(info =>
+        info.type !== 'browser_ui' && info.browserContextId === context.browserContextId,
+      ),
+    )
+    if (!contexts.length) return
+    const socket = browser.socket
+    return browser.context_checking = (async () => {
+      do {
+        browser.context_check_pending = false
+        const { targetInfos } = await socket.req({ method: 'Target.getTargets' })
+        if (browser.socket !== socket || browser.closing) return
+        for (const context of Object.values(browser.contexts)) {
+          if (!context.browserContextId || context.connecting || context.closing) continue
+          const belongs = info => info.type !== 'browser_ui' && info.browserContextId === context.browserContextId
+          if (targetInfos.some(belongs) || Object.values(browser.target_info).some(belongs)) continue
+          if (Object.values(browser.targets).some(target =>
+            target.connecting && (target.context?.name ?? target.context) === context.name,
+          )) continue
+          await this._dispose_context(browser, context)
+        }
+      } while (browser.context_check_pending)
+    })().catch(error => util.emit(this, 'error', {
+      browser: browser.name,
+      error,
+    })).finally(() => {
+      browser.context_checking = null
+    })
   }
 
   async _connect(record) {
+    if (record.reverse || record.cdp_ext) {
+      await this.server_closing
+      if (!this.server) this.listen_reverse()
+    }
     let url = record.websocket_url
-    if (!url && (record.http_url || record.port)) {
+    if (!record.reverse && !record.cdp_ext && !url && (record.http_url || record.port)) {
       const discovery = new URL(record.http_url || 'http://' + (record.host || '127.0.0.1') + ':' + record.port)
       if (!discovery.pathname.endsWith('/json/version')) discovery.pathname = discovery.pathname.replace(/\/$/, '') + '/json/version'
       const response = await fetch(discovery, { signal: AbortSignal.timeout(record.connect_timeout_ms ?? 15000) })
       if (!response.ok) throw new Error('CDP discovery: HTTP ' + response.status)
       url = (await response.json()).webSocketDebuggerUrl
     }
-    if (!url) {
-      if (record.proc?.exitCode === null && record.proc.signalCode === null) url = record.socket.url
+    if (record.cdp_ext || (!record.reverse && !url)) {
+      if (record.proc?.exitCode === null && record.proc.signalCode === null) url = record.socket?.url
       else {
+        const args = browser.build_args(record)
+        if (record.cdp_ext) {
+          record.extension_path = path.resolve(
+            args['user-data-dir'] || path.join(this.base_path, record.name),
+            args['profile-directory'] || 'Default',
+            'cdp_ext',
+          )
+          await fs.mkdir(record.extension_path, { recursive: true })
+          record.extension_path = await fs.realpath(record.extension_path)
+          record.extension_id = browser.extension_id(record.extension_path)
+          const manifest = JSON.parse(await fs.readFile(new URL('./cdp_ext/manifest.json', import.meta.url), 'utf8'))
+          let host = this.server.addr.hostname
+          if (host === '0.0.0.0') host = '127.0.0.1'
+          if (host === '::') host = '::1'
+          manifest.cdp.websocket_url = 'ws://' + (host.includes(':') ? '[' + host + ']' : host) + ':' + this.server.addr.port
+          await fs.writeFile(path.join(record.extension_path, 'manifest.json'), JSON.stringify(manifest))
+          await fs.copyFile(new URL('./cdp_ext/cdp_ext.js', import.meta.url), path.join(record.extension_path, 'cdp_ext.js'))
+        }
+        if (args['user-data-dir'] && record.update_preferences !== false) {
+          await browser.update_profile_preferences(path.join(args['user-data-dir'], args['profile-directory'] || 'Default', 'Preferences'), {
+            'translate.enabled': record.translations === true,
+            'signin.allowed': record.login === true,
+            'signin.allowed_on_next_startup': record.login === true,
+            ...record.preferences,
+          })
+          if (Object.keys(record.local_state).length) {
+            await browser.update_profile_preferences(path.join(args['user-data-dir'], 'Local State'), record.local_state)
+          }
+        }
         const launched = await browser.launch(record)
         record.proc = launched.proc
+        if (record.cdp_ext) record.proc.once('exit', () => this._close_reverse_if_idle())
         url = launched.url
       }
     }
+    if (record.reverse || record.cdp_ext) {
+      const id = record.extension_id ?? record.name
+      url = this.reverse_sockets[id]
+      if (url?.readyState !== WebSocket.OPEN) {
+        url = (await util.on_first(this, 'reverse_' + encodeURIComponent(id), record.connect_timeout_ms ?? 15000)).args[0].detail
+      }
+    }
     const socket = record.socket = new jsrpc(url, record)
     record.target_info = {}
     record.session_targets = {}
@@ -380,7 +661,9 @@ export class cdp extends EventTarget {
         code: event.code,
       })
     })
-    await util.on_first(socket, 'open error close', record.connect_timeout_ms ?? 15000)
+    if (socket.readyState !== WebSocket.OPEN) {
+      await util.on_first(socket, 'open error close', record.connect_timeout_ms ?? 15000)
+    }
     await socket.req({
       method: 'Target.setAutoAttach',
       params: {
@@ -393,21 +676,50 @@ export class cdp extends EventTarget {
       method: 'Target.setDiscoverTargets',
       params: { discover: true },
     })
+    if (Object.values(record.contexts).some(context => context.browserContextId)) {
+      const { browserContextIds } = await socket.req({ method: 'Target.getBrowserContexts' })
+      for (const context of Object.values(record.contexts)) {
+        if (!browserContextIds.includes(context.browserContextId)) context.browserContextId = null
+      }
+    }
   }
 
   async _target(browser, target) {
     const socket = browser.socket
+    const reverse = browser.reverse || browser.cdp_ext
+    if (target.incognito && !reverse && target.context == null && target.create_params?.browserContextId == null) {
+      target.context = 'incognito'
+    }
+    if (target.context != null && target.create_params?.browserContextId != null) {
+      throw new Error('Use target.context or create_params.browserContextId, not both')
+    }
+    const context = target.context == null ? null : await this._context(browser, target.context)
+    if (context) target.context = context.name
     if (!browser.target_info[target.targetId]) {
       target.targetId = (await socket.req({
         method: 'Target.createTarget',
         params: {
           url: 'about:blank',
           background: true,
+          browserContextId: context?.browserContextId,
+          ...(reverse && target.incognito ? { incognito: true } : {}),
           ...target.create_params,
         },
       })).targetId
+      if (context) (browser.target_info[target.targetId] ??= {}).browserContextId = context.browserContextId
     }
     const info = browser.target_info[target.targetId] ??= {}
+    if (context && info.browserContextId !== context.browserContextId) {
+      throw new Error('Existing target belongs to a different browser context')
+    }
+    if (reverse && target.incognito && !info.incognito) {
+      const { targetInfo } = await socket.req({
+        method: 'Target.getTargetInfo',
+        params: { targetId: target.targetId },
+      })
+      Object.assign(info, targetInfo)
+      if (!info.incognito) throw new Error('Existing target is not incognito')
+    }
     if (!info.sessionId) {
       info.sessionId = (await socket.req({
         method: 'Target.attachToTarget',
@@ -478,15 +790,25 @@ export class cdp extends EventTarget {
         ...this.options.preferences,
         ...spec.preferences,
       },
+      local_state: {
+        ...this.options.local_state,
+        ...spec.local_state,
+      },
       targets: Object.create(null),
+      contexts: Object.create(null),
+    }
+    if (target?.context != null && (record.reverse || record.cdp_ext)) {
+      throw new Error('Named browser contexts are unavailable through the reverse Chrome extension')
     }
     if (!record.connecting && record.socket?.readyState !== WebSocket.OPEN) {
       record.connecting = this._connect(record).catch(error => {
         record.socket?.close()
-        record.proc?.kill()
+        if (!record.cdp_ext) record.proc?.kill()
         throw error
-      }).finally(() => {
+      }).finally(async () => {
         record.connecting = null
+        await this._close_reverse_if_idle()
+        this._close_contexts_if_idle(record)
       })
     }
     await record.connecting
@@ -503,10 +825,15 @@ export class cdp extends EventTarget {
       if (!entry.sessionId) {
         entry.connecting ??= this._target(record, entry).finally(() => {
           entry.connecting = null
+          this._close_contexts_if_idle(record)
         })
         await entry.connecting
       }
     }
+    if (method === 'Target.disposeBrowserContext' && target == null) {
+      const context = Object.values(record.contexts).find(value => value.browserContextId === native.browserContextId)
+      if (context) return this._dispose_context(record, context, native)
+    }
     const result = await record.socket.req({
       method,
       params: native,
@@ -516,23 +843,39 @@ export class cdp extends EventTarget {
     if (method === 'Target.closeTarget' && result.success) {
       this._detach(record, record.target_info[native.targetId]?.sessionId)
       delete record.target_info[native.targetId]
+      await this._close_contexts_if_idle(record)
     }
     return result
   }
 
   async close() {
-    await Promise.all(Object.values(this.browsers).map(async browser => {
-      try {
-        await browser.connecting
-        if (browser.proc && browser.socket?.readyState === WebSocket.OPEN) {
-          await browser.socket.req({ method: 'Browser.close' })
+    try {
+      await Promise.all(Object.values(this.browsers).map(async browser => {
+        try {
+          await browser.connecting
+          browser.closing = true
+          await Promise.allSettled(Object.values(browser.targets).map(target => target.connecting))
+          await browser.context_checking
+          for (const context of Object.values(browser.contexts)) {
+            await context.closing
+            if (context.browserContextId && browser.socket?.readyState === WebSocket.OPEN) {
+              await this._dispose_context(browser, context)
+            }
+          }
+          if (browser.proc && !browser.cdp_ext && browser.socket?.readyState === WebSocket.OPEN) {
+            await browser.socket.req({ method: 'Browser.close' })
+          }
+        } finally {
+          browser.socket?.close()
+          browser.proc?.kill()
         }
-      } finally {
-        browser.socket?.close()
-        browser.proc?.kill()
-      }
-    }))
-    this.browsers = Object.create(null)
+      }))
+    } finally {
+      this.browsers = Object.create(null)
+      for (const socket of Object.values(this.reverse_sockets)) socket.close()
+      this.reverse_sockets = Object.create(null)
+      await this._close_reverse_if_idle()
+    }
   }
 }