unmagic-browser 0.1.0

data/lib/unmagic/browser/driver/cloudflare/transport.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "puppeteer"
+require "async/websocket/client"
+
+module Unmagic
+  class Browser
+    module Driver
+      class Cloudflare < Base
+        # The one seam between puppeteer-ruby and Cloudflare.
+        #
+        # puppeteer-ruby opens the CDP websocket with no custom headers, and
+        # Cloudflare's remote devtools endpoint authenticates on the handshake
+        # with `Authorization: Bearer`. This subclass injects it; everything
+        # else about driving the browser is unchanged.
+        class Transport < Puppeteer::WebSocketTransport
+          class << self
+            # Connect to Cloudflare's devtools endpoint.
+            #
+            # Must be called from inside a running Async reactor. Outside one,
+            # `connect`'s `Async do ... end` becomes the reactor itself and
+            # blocks until the read loop ends — that is, until Cloudflare drops
+            # the idle socket — handing back a transport that's already dead.
+            #
+            # @param url [String] the `wss://` devtools endpoint
+            # @param token [String] a Cloudflare API token with Browser Rendering access
+            # @return [Transport] a connected transport
+            def create(url, token)
+              new(url, token).tap { |transport| transport.connect.wait }
+            end
+          end
+
+          # @param url [String] the `wss://` devtools endpoint
+          # @param token [String] a Cloudflare API token with Browser Rendering access
+          def initialize(url, token)
+            super(url)
+            @headers = [["authorization", "Bearer #{token}"]]
+          end
+
+          # @return [Async::Promise] resolved once the socket is open
+          def connect
+            return @connect_promise if @connect_promise
+
+            @connect_promise = Async::Promise.new
+            @task = Async do
+              Async::WebSocket::Client.connect(@endpoint, headers: @headers) do |connection|
+                @connection = connection
+                @connected = true
+                @connect_promise.resolve(true) unless @connect_promise.resolved?
+                send(:receive_loop, connection)
+              end
+            rescue Async::Stop
+              nil # Task stopped on the way out; nothing to clean up here.
+            rescue StandardError => error
+              @connect_promise.reject(error) unless @connect_promise.resolved?
+              close
+            ensure
+              @connected = false
+            end
+
+            @connect_promise
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unmagic/browser/driver/cloudflare.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "puppeteer"
+require "uri"
+
+module Unmagic
+  class Browser
+    module Driver
+      # [Cloudflare Browser
+      # Rendering](https://developers.cloudflare.com/browser-rendering/) — no
+      # browser runs on your own servers.
+      #
+      # It's the only driver with two ways in, and it uses both. A one-off page
+      # grab goes to the REST API, which renders and returns in one request and
+      # holds no session at all. A {Session} connects over CDP to a real remote
+      # browser, which is what multi-step work needs — and what occupies one of
+      # your account's concurrent slots until it's closed.
+      #
+      # @example
+      #   Unmagic::Browser.new(driver: Unmagic::Browser::Driver::Cloudflare.new(
+      #     account_id: ENV["CLOUDFLARE_ACCOUNT_ID"],
+      #     token:      ENV["CLOUDFLARE_API_TOKEN"],
+      #   ))
+      #
+      # @example Both credentials come from the environment
+      #   Unmagic::Browser.new(driver: :cloudflare)
+      class Cloudflare < Base
+        # Where the REST rendering endpoints live.
+        API_ROOT = "https://api.cloudflare.com/client/v4/accounts/%<account_id>s/browser-rendering"
+
+        # Cloudflare's own devtools endpoint. Closing the browser rather than
+        # merely disconnecting is what frees the concurrency slot immediately;
+        # an abandoned session holds one until the service's own idle timeout
+        # reaps it.
+        CDP_ENDPOINT = "wss://api.cloudflare.com/client/v4/accounts/%<account_id>s/browser-rendering/devtools/browser"
+
+        # Seconds to wait on a REST render before giving up. Cloudflare's own
+        # ceiling is lower than this; the margin is for the round trip.
+        REST_TIMEOUT = 60
+
+        # @return [String] the Cloudflare account the browsers belong to
+        attr_reader :account_id
+
+        # Where the token comes from when it isn't passed, most specific first.
+        # `CLOUDFLARE_API_TOKEN` is Cloudflare's own name for it — the one
+        # wrangler and their docs use — so it's what a machine already set up
+        # for Cloudflare will have. The narrower name wins when both are set, so
+        # a token scoped to just Browser Rendering can be kept apart from the
+        # account-wide one.
+        TOKEN_VARIABLES = ["CLOUDFLARE_BROWSER_RENDERING_TOKEN", "CLOUDFLARE_API_TOKEN"].freeze
+
+        # @param account_id [String, nil] defaults to `$CLOUDFLARE_ACCOUNT_ID`
+        # @param token [String, nil] an API token with Browser Rendering access,
+        #   defaulting to the first of {TOKEN_VARIABLES} that's set
+        # @raise [Error::Misconfigured] when either credential is missing
+        def initialize(account_id: nil, token: nil)
+          super()
+          @account_id = account_id || ENV["CLOUDFLARE_ACCOUNT_ID"]
+          @token = token || TOKEN_VARIABLES.filter_map { |name| presence(ENV[name]) }.first
+          require_credential(@account_id, "account_id", ["CLOUDFLARE_ACCOUNT_ID"])
+          require_credential(@token, "token", TOKEN_VARIABLES)
+        end
+
+        # @param url [String] the page to render
+        # @return [String] the page as Markdown, rendered by Cloudflare
+        def markdown(url)
+          json_request("markdown", url: url)
+        end
+
+        # @param url [String] the page to render
+        # @return [String] the page's HTML, after JavaScript has run
+        def html(url)
+          json_request("content", url: url)
+        end
+
+        # @param url [String] the page to render
+        # @param full_page [Boolean] capture the whole scrollable page
+        # @param selector [String, nil] capture just the first element matching this CSS
+        # @return [String] PNG bytes
+        def screenshot(url, full_page: false, selector: nil)
+          binary_request(
+            "screenshot",
+            url: url,
+            selector: selector,
+            screenshotOptions: { fullPage: full_page, type: "png" },
+          )
+        end
+
+        # @param url [String] the page to render
+        # @return [String] PDF bytes
+        def pdf(url)
+          binary_request("pdf", url: url)
+        end
+
+        # Nothing to tear down: one-offs never open a browser, and every session
+        # owns its own.
+        #
+        # @return [void]
+        def close
+          nil
+        end
+
+        protected
+
+        # Cloudflare needs its auth header on the websocket handshake, so the
+        # transport has to be built before the connection — and inside a
+        # reactor, or its read loop would take over the calling thread. We run
+        # our own so both halves live in the same one, then hand back the same
+        # kind of thread-safe proxy `Puppeteer.connect` would have.
+        def connect_browser
+          runner = Puppeteer::ReactorRunner.new
+          begin
+            browser = runner.sync do
+              transport = Transport.create(format(CDP_ENDPOINT, account_id: @account_id), @token)
+              Puppeteer.connect(transport: transport)
+            end
+          rescue StandardError
+            runner.close
+            raise
+          end
+
+          Puppeteer::ReactorRunner::Proxy.new(runner, browser, owns_runner: true)
+        end
+
+        # Disconnecting alone leaves the remote browser running, idle, holding a
+        # concurrency slot until Cloudflare's own timeout reaps it. Closing it
+        # gives the slot back now.
+        def disconnect_browser(browser)
+          browser.close
+        rescue StandardError
+          nil
+        end
+
+        private
+
+        def require_credential(value, option, variables)
+          return if presence(value)
+
+          names = variables.map { |name| "$#{name}" }.join(" or ")
+          raise Error::Misconfigured,
+            "Cloudflare Browser Rendering needs #{option}: pass Cloudflare.new(#{option}: ...) or set #{names}."
+        end
+
+        def presence(value)
+          value unless value.nil? || value.to_s.strip.empty?
+        end
+
+        def json_request(endpoint, **body)
+          response = post(endpoint, body)
+          payload = parse_json(response)
+          raise Error::RenderFailed, "Cloudflare couldn't render #{body[:url]}." unless payload["success"]
+
+          payload["result"].to_s
+        end
+
+        def binary_request(endpoint, **body)
+          post(endpoint, body).body
+        end
+
+        def post(endpoint, body)
+          uri = URI.join("#{format(API_ROOT, account_id: @account_id)}/", endpoint)
+          request = Net::HTTP::Post.new(uri)
+          request["Authorization"] = "Bearer #{@token}"
+          request["Content-Type"] = "application/json"
+          request.body = JSON.generate(body.compact)
+
+          check(perform(request, uri), body[:url])
+        end
+
+        def perform(request, uri)
+          Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 10, read_timeout: REST_TIMEOUT) do |http|
+            http.request(request)
+          end
+        rescue Net::OpenTimeout, Net::ReadTimeout => error
+          raise Error::TimedOut, "Cloudflare Browser Rendering took too long: #{error.message}"
+        rescue SystemCallError, SocketError, OpenSSL::SSL::SSLError, IOError => error
+          raise Error::Unreachable, "Couldn't reach Cloudflare Browser Rendering: #{error.message}"
+        end
+
+        def check(response, url)
+          return response if response.is_a?(Net::HTTPSuccess)
+
+          case response
+          when Net::HTTPTooManyRequests
+            raise Error::RateLimited, "Cloudflare Browser Rendering rate limit exceeded."
+          when Net::HTTPUnauthorized, Net::HTTPForbidden
+            raise Error::Misconfigured, "Cloudflare rejected the Browser Rendering credentials (HTTP #{response.code})."
+          else
+            raise Error::RenderFailed, failure_message(response, url)
+          end
+        end
+
+        def failure_message(response, url)
+          detail = begin
+            JSON.parse(response.body).dig("errors", 0, "message")
+          rescue StandardError
+            nil
+          end
+          "Cloudflare couldn't render #{url} (HTTP #{response.code}#{" — #{detail}" if detail})."
+        end
+
+        def parse_json(response)
+          JSON.parse(response.body)
+        rescue JSON::ParserError => error
+          raise Error::RenderFailed, "Cloudflare Browser Rendering returned something that isn't JSON: #{error.message}"
+        end
+      end
+    end
+  end
+end
+
+require_relative "cloudflare/transport"

data/lib/unmagic/browser/driver/connection.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Unmagic
+  class Browser
+    module Driver
+      # One live browser, and the driver's promise to shut it down properly.
+      #
+      # Every {Session} owns exactly one of these — that's what "a kept session
+      # holds a real browser, and on Cloudflare one of a limited number of
+      # concurrent slots, until you close it" means in code.
+      class Connection
+        # @param driver [Base] the driver that opened this browser
+        # @param browser [Puppeteer::Browser] the connected browser
+        def initialize(driver, browser)
+          @driver = driver
+          @browser = browser
+          @closed = false
+          @mutex = Mutex.new
+        end
+
+        # @return [Puppeteer::Browser] the live browser
+        attr_reader :browser
+
+        # A page to drive.
+        #
+        # A freshly started browser already has one blank tab open, and opening
+        # a second costs most of a second — so the first caller gets the one
+        # that's already there.
+        #
+        # @return [Puppeteer::Page]
+        def new_page
+          raise Error::SessionClosed, "This browser connection is closed." if closed?
+
+          Failures.wrap("opening a page") { claim_initial_page || @browser.new_page }
+        end
+
+        # Shut the browser down the way its driver wants: killed, disconnected
+        # from, or told to close so a remote slot is freed immediately.
+        #
+        # @return [void]
+        def close
+          @mutex.synchronize do
+            return if @closed
+
+            @closed = true
+            @driver.send(:disconnect_browser, @browser)
+          end
+        end
+
+        # @return [Boolean] whether this connection has been closed
+        def closed?
+          @closed
+        end
+
+        private
+
+        # The blank tab a new browser starts with, once and only once, and only
+        # while it's still blank and alone — anything else and we'd be handing
+        # one caller another's page.
+        def claim_initial_page
+          @mutex.synchronize do
+            return nil if @initial_page_claimed
+
+            @initial_page_claimed = true
+          end
+
+          pages = @browser.pages
+          pages.first if pages.size == 1 && pages.first.url == "about:blank"
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/unmagic/browser/driver/local.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "puppeteer"
+
+module Unmagic
+  class Browser
+    module Driver
+      # Chrome on this machine.
+      #
+      # The development default, and the only driver where you can watch what's
+      # happening: `headless: false` opens a real window, which is how a person
+      # gets to solve a captcha mid-session and hand control back.
+      #
+      # @example
+      #   Unmagic::Browser.new(driver: Unmagic::Browser::Driver::Local.new(headless: false))
+      class Local < Base
+        # Chrome flags that only make sense for automation. `--no-sandbox` is
+        # what lets Chrome run as root inside a container, which is where CI
+        # lives.
+        DEFAULT_ARGS = [
+          "--disable-dev-shm-usage",
+          "--no-sandbox",
+          "--disable-gpu",
+        ].freeze
+
+        # @return [Boolean] whether Chrome runs without a window
+        attr_reader :headless
+
+        # @return [String, nil] the Chrome binary to run, or nil to let Puppeteer find one
+        attr_reader :executable_path
+
+        # @return [Array<String>] the flags Chrome is launched with
+        attr_reader :args
+
+        # @param headless [Boolean] false opens a window you can watch and click
+        # @param executable_path [String, nil] a specific Chrome binary
+        # @param args [Array<String>] extra Chrome flags, added to {DEFAULT_ARGS}
+        # @param slow_mo [Integer, nil] milliseconds to pause between operations, to watch a run
+        def initialize(headless: true, executable_path: nil, args: [], slow_mo: nil)
+          super()
+          @headless = headless
+          @executable_path = executable_path
+          @args = (DEFAULT_ARGS + args).uniq.freeze
+          @slow_mo = slow_mo
+        end
+
+        protected
+
+        def connect_browser
+          Puppeteer.launch(**launch_options)
+        rescue StandardError => error
+          raise Error::Misconfigured, chrome_missing_message(error) if chrome_missing?(error)
+
+          raise
+        end
+
+        # Local Chrome is ours: we started it, so we kill it. Anything less
+        # leaves a browser process behind for every session ever opened.
+        def disconnect_browser(browser)
+          browser.close
+        rescue StandardError
+          nil
+        end
+
+        private
+
+        def launch_options
+          {
+            headless: @headless,
+            args: @args,
+            executable_path: @executable_path,
+            slow_mo: @slow_mo,
+          }.compact
+        end
+
+        def chrome_missing?(error)
+          error.message.to_s.match?(/Failed to launch|executable|ENOENT|Could not find/i)
+        end
+
+        def chrome_missing_message(error)
+          "Couldn't start local Chrome (#{error.message}). Install Chrome, or point the driver at a " \
+            "binary with Unmagic::Browser::Driver::Local.new(executable_path: \"...\")."
+        end
+      end
+    end
+  end
+end

data/lib/unmagic/browser/driver/remote.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "puppeteer"
+
+module Unmagic
+  class Browser
+    module Driver
+      # A browser you already run somewhere else — browserless, a pool of your
+      # own, a Chrome started with `--remote-debugging-port`.
+      #
+      # @example
+      #   Unmagic::Browser.new(driver: Unmagic::Browser::Driver::Remote.new(ws: "ws://browserless.internal:3000"))
+      class Remote < Base
+        # @return [String] the websocket endpoint this driver connects to
+        attr_reader :ws
+
+        # @param ws [String] a CDP websocket URL, e.g. `"ws://browserless.internal:3000"`
+        def initialize(ws: nil)
+          super()
+          @ws = ws || ENV["BROWSER_WS_ENDPOINT"]
+          return if @ws
+
+          raise Error::Misconfigured,
+            "The remote driver needs a websocket endpoint: Remote.new(ws: \"ws://...\") or $BROWSER_WS_ENDPOINT."
+        end
+
+        protected
+
+        def connect_browser
+          Puppeteer.connect(browser_ws_endpoint: @ws)
+        end
+
+        # The browser on the other end isn't ours to kill — it may be a pool
+        # serving other callers. We let go of it and leave it running; a service
+        # that hands out a browser per connection (browserless does) reaps it
+        # itself once we're gone.
+        def disconnect_browser(browser)
+          browser.disconnect
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/unmagic/browser/driver.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Unmagic
+  class Browser
+    # The three ways to get a browser: one on this machine, one somewhere else
+    # you already run, or Cloudflare's.
+    #
+    # They're interchangeable — the same {Session} verbs run against all three —
+    # so picking one is the only decision, and it's made once, when you
+    # construct the browser.
+    module Driver
+      # What a bare symbol means.
+      REGISTRY = {
+        local: "Unmagic::Browser::Driver::Local",
+        remote: "Unmagic::Browser::Driver::Remote",
+        cloudflare: "Unmagic::Browser::Driver::Cloudflare",
+      }.freeze
+
+      class << self
+        # Coerce whatever a caller passed as `driver:` into a driver.
+        #
+        # A symbol builds the driver with its defaults; anything else is taken
+        # as a driver already, which is how a driver gets options — and how
+        # somebody's own driver gets used without inheriting from {Base}.
+        #
+        # @param value [Symbol, String, Base] `:local`, `:remote`, `:cloudflare`, or an instance
+        # @return [Base]
+        # @raise [Error::Misconfigured] when the symbol names no known driver
+        def build(value)
+          return value unless value.is_a?(Symbol) || value.is_a?(String)
+
+          name = value.to_sym
+          class_name = REGISTRY[name]
+          unless class_name
+            known = REGISTRY.keys.map(&:inspect).join(", ")
+            raise Error::Misconfigured, "Unknown driver #{value.inspect}. Known drivers are #{known}."
+          end
+
+          Object.const_get(class_name).new
+        end
+
+        # Remember a driver so its shared one-off connection is torn down when
+        # the process ends. Registering is idempotent, and the hook is installed
+        # only once a driver has actually opened a browser — a program that only
+        # ever talks to Cloudflare's REST API never installs one.
+        #
+        # @param driver [Base]
+        # @return [void]
+        def close_at_exit(driver)
+          @registry_mutex ||= Mutex.new
+          @registry_mutex.synchronize do
+            @registry ||= []
+            return if @registry.include?(driver)
+
+            install_at_exit_hook unless @at_exit_installed
+            @at_exit_installed = true
+            @registry << driver
+          end
+        end
+
+        private
+
+        def install_at_exit_hook
+          at_exit do
+            @registry.each do |driver|
+              driver.close
+            rescue StandardError
+              nil # Exiting anyway; a browser we can't reach is already gone.
+            end
+          end
+        end
+      end
+    end
+  end
+end