unmagic-browser 0.1.0

data/lib/unmagic/browser/element.rb

@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+module Unmagic
+  class Browser
+    # One element on the page, found by a {Locator}.
+    #
+    # Everything a {Session} verb does to a single element goes through here, so
+    # `session.click_button "Go"` and `session.find("#go").click` take exactly
+    # the same path. Like a session, an element serialises its operations
+    # against the session it came from.
+    #
+    # @example
+    #   session.all(".product").map { |element| element.text }
+    class Element
+      # @param session [Session] the session this element was found in
+      # @param handle [Puppeteer::ElementHandle] the live handle in the page
+      def initialize(session, handle)
+        @session = session
+        @handle = handle
+      end
+
+      # The live Puppeteer handle, for work this API doesn't wrap.
+      #
+      # @return [Puppeteer::ElementHandle]
+      attr_reader :handle
+      alias_method :driver, :handle
+
+      # @return [String] the element's visible text, whitespace collapsed
+      def text
+        evaluate("element => (element.innerText === undefined ? element.textContent : element.innerText)")
+          .to_s.gsub(/\s+/, " ").strip
+      end
+
+      # @return [String] the element's own HTML, tag included
+      def html
+        evaluate("element => element.outerHTML")
+      end
+
+      # @return [String] the element's inner HTML
+      def inner_html
+        evaluate("element => element.innerHTML")
+      end
+
+      # @return [String] the lowercased tag name, e.g. `"input"`
+      def tag_name
+        evaluate("element => element.tagName.toLowerCase()")
+      end
+
+      # The element's current value — what's typed into a field, what's selected
+      # in a `<select>`, or nil for anything without one.
+      #
+      # @return [String, nil]
+      def value
+        evaluate("element => ('value' in element ? element.value : null)")
+      end
+
+      # @param name [String, Symbol] the attribute name
+      # @return [String, nil] the attribute's value, or nil when it isn't set
+      def attribute(name)
+        evaluate("(element, name) => element.getAttribute(name)", name.to_s)
+      end
+      alias_method :[], :attribute
+
+      # @return [Boolean] whether the element is rendered and not hidden
+      def visible?
+        evaluate(<<~JAVASCRIPT)
+          (element) => {
+            if (element.hidden) return false;
+            const style = getComputedStyle(element);
+            if (style.visibility === "hidden" || style.display === "none") return false;
+            return element.getClientRects().length > 0;
+          }
+        JAVASCRIPT
+      end
+
+      # @return [Boolean] whether a checkbox or radio is ticked
+      def checked?
+        !!evaluate("element => !!element.checked")
+      end
+
+      # @return [Boolean] whether the element is disabled
+      def disabled?
+        !!evaluate("element => !!element.disabled")
+      end
+
+      # Click the element, scrolling it into view first.
+      #
+      # @return [self]
+      def click
+        act do
+          @handle.scroll_into_view_if_needed
+          @handle.click
+        end
+      end
+
+      # Move the pointer over the element.
+      #
+      # @return [self]
+      def hover
+        act do
+          @handle.scroll_into_view_if_needed
+          @handle.hover
+        end
+      end
+
+      # Replace whatever's in the field with +value+.
+      #
+      # The existing content is selected and deleted rather than assigned over,
+      # so frameworks that watch for real key events (React and friends) see the
+      # change.
+      #
+      # @param value [String] the text to type
+      # @return [self]
+      def fill_in(value)
+        act do
+          @handle.scroll_into_view_if_needed
+          @handle.focus
+          @handle.evaluate(<<~JAVASCRIPT)
+            (element) => {
+              if (element.select) { element.select(); return; }
+              const range = document.createRange();
+              range.selectNodeContents(element);
+              const selection = getSelection();
+              selection.removeAllRanges();
+              selection.addRange(range);
+            }
+          JAVASCRIPT
+          @handle.press("Backspace")
+          @handle.type_text(value.to_s) unless value.to_s.empty?
+        end
+      end
+
+      # Pick an option in a `<select>` by its visible label, or failing that its
+      # value — a caller reading the page sees labels, not values.
+      #
+      # @param option [String] the option's label or value
+      # @return [self]
+      # @raise [Error::NotFound] when no option matches
+      def select_option(option)
+        act do
+          found = @handle.evaluate(<<~JAVASCRIPT, option.to_s)
+            (element, wanted) => {
+              if (element.tagName.toLowerCase() !== "select") throw new Error("not a <select>");
+              const norm = (value) => (value || "").replace(/\\s+/g, " ").trim().toLowerCase();
+              const needle = norm(wanted);
+              const options = Array.from(element.options);
+              const match = options.find((option) => norm(option.label || option.textContent) === needle) ||
+                            options.find((option) => norm(option.value) === needle) ||
+                            options.find((option) => norm(option.label || option.textContent).indexOf(needle) !== -1);
+              if (!match) return false;
+              if (element.multiple) { match.selected = true; } else { element.value = match.value; }
+              element.dispatchEvent(new Event("input", { bubbles: true }));
+              element.dispatchEvent(new Event("change", { bubbles: true }));
+              return true;
+            }
+          JAVASCRIPT
+          raise Error::NotFound, "No option matching #{option.inspect} in this select." unless found
+        end
+      end
+
+      # Tick a checkbox, or leave it ticked if it already is.
+      #
+      # @return [self]
+      def check
+        set_checked(true)
+      end
+
+      # Untick a checkbox, or leave it unticked if it already is.
+      #
+      # @return [self]
+      def uncheck
+        set_checked(false)
+      end
+
+      # Attach files to a file input.
+      #
+      # @param paths [Array<String>] paths to the files
+      # @return [self]
+      def attach(*paths)
+        act { @handle.upload_file(*paths.flatten.map(&:to_s)) }
+      end
+
+      # Press a key with this element focused.
+      #
+      # @param key [String, Symbol] a key name, e.g. `:enter` or `"ArrowDown"`
+      # @return [self]
+      def press(key)
+        act { @handle.press(Session.key_name(key)) }
+      end
+
+      # @return [String] PNG bytes of just this element
+      def screenshot
+        @session.synchronize do
+          @handle.scroll_into_view_if_needed
+          @handle.screenshot(type: "png")
+        end
+      end
+
+      # Find one element inside this one. Same locator rules as {Session#find}.
+      #
+      # @param query [String, nil] see {Locator.build}
+      # @param options [Hash] see {Locator.build}
+      # @return [Element]
+      # @raise [Error::NotFound] when nothing matches in time
+      def find(query = nil, **options)
+        @session.find(query, scope: self, **options)
+      end
+
+      # Find every element inside this one. Same locator rules as {Session#all}.
+      #
+      # @param query [String, nil] see {Locator.build}
+      # @param options [Hash] see {Locator.build}
+      # @return [Array<Element>]
+      def all(query = nil, **options)
+        @session.all(query, scope: self, **options)
+      end
+
+      # Run JavaScript with this element as the first argument.
+      #
+      # @param script [String] a JavaScript function body, e.g. `"el => el.dataset.id"`
+      # @param args [Array] extra arguments passed after the element
+      # @return [Object] whatever the script returns, as a Ruby value
+      def evaluate(script, *args)
+        @session.synchronize { @handle.evaluate(script, *args) }
+      end
+
+      # Release the handle. Elements found by a session are disposed with the
+      # page, so this is only worth calling in a long loop.
+      #
+      # @return [void]
+      def dispose
+        @session.synchronize { @handle.dispose }
+      end
+
+      # @return [String] the element and the handle behind it
+      def inspect
+        "#<#{self.class.name} #{@handle.inspect}>"
+      end
+
+      private
+
+      def set_checked(wanted)
+        act do
+          @handle.scroll_into_view_if_needed
+          @handle.click if @handle.evaluate("element => !!element.checked") != wanted
+        end
+      end
+
+      # Every action returns the element so calls chain, and every action holds
+      # the session lock so two callers can't interleave a keystroke.
+      def act
+        @session.synchronize { yield }
+        self
+      end
+    end
+  end
+end

data/lib/unmagic/browser/emulation.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+require "puppeteer"
+
+module Unmagic
+  class Browser
+    # Everything about *how* a page renders — colour scheme, viewport, device,
+    # locale, timezone, geolocation, media type — in one immutable value object.
+    #
+    # It's the only place emulation lives, and it's set in exactly two places:
+    # passed to {Unmagic::Browser#open}, or changed live with
+    # {Session#emulate}.
+    #
+    # Anything left unset renders at a *fixed* default — a plain desktop
+    # viewport, light, screen media — rather than at whatever the machine
+    # underneath happens to prefer. That's what makes re-applying an older
+    # Emulation a clean revert rather than a patch, and what stops the same page
+    # rendering differently on a laptop and on CI.
+    #
+    # @example
+    #   Unmagic::Browser::Emulation.new(color_scheme: :dark, device: "iPhone 13")
+    class Emulation
+      # What a page looks like with nothing emulated. Applied for any viewport
+      # key the caller leaves out, so reverting is always total.
+      DEFAULT_VIEWPORT = {
+        width: 1280,
+        height: 800,
+        scale: 1,
+        mobile: false,
+        touch: false,
+      }.freeze
+
+      # What `prefers-color-scheme` can be told to report.
+      COLOR_SCHEMES = [:light, :dark, :no_preference].freeze
+
+      # What `prefers-reduced-motion` can be told to report.
+      REDUCED_MOTIONS = [:reduce, :no_preference].freeze
+
+      # What the page can be told it's being rendered for.
+      MEDIA_TYPES = [:screen, :print].freeze
+
+      # Present as a current desktop Chrome rather than as a headless build:
+      # pages render the way a real visitor sees them, and modern-browser gates
+      # let us through.
+      DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " \
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+
+      class << self
+        # Coerce whatever a caller passed as `emulate:` into an Emulation.
+        #
+        # @param value [Emulation, Hash, nil] an instance, a hash of knobs, or nothing
+        # @return [Emulation]
+        # @raise [ArgumentError] when +value+ is neither
+        def build(value)
+          case value
+          when nil then new
+          when Emulation then value
+          when Hash then new(**value)
+          else
+            raise ArgumentError, "emulate: expects an Unmagic::Browser::Emulation or a Hash, got #{value.class}"
+          end
+        end
+      end
+
+      attr_reader :color_scheme, :reduced_motion, :viewport, :device, :user_agent,
+        :locale, :timezone, :geolocation, :media
+
+      # @param color_scheme [Symbol, nil] `:light`, `:dark` or `:no_preference`
+      # @param reduced_motion [Symbol, nil] `:reduce` or `:no_preference`
+      # @param viewport [Hash, nil] `width:`, `height:`, `scale:`, `mobile:`, `touch:`
+      # @param device [String, nil] a Puppeteer device name, e.g. `"iPhone 13"`
+      # @param user_agent [String, nil] overrides the device's own user agent
+      # @param locale [String, nil] a BCP 47 tag, sent as `Accept-Language`
+      # @param timezone [String, nil] an IANA zone, e.g. `"Australia/Sydney"`
+      # @param geolocation [Hash, nil] `latitude:`, `longitude:` and optionally `accuracy:`
+      # @param media [Symbol, nil] `:screen` or `:print`
+      def initialize(color_scheme: nil, reduced_motion: nil, viewport: nil, device: nil,
+        user_agent: nil, locale: nil, timezone: nil, geolocation: nil, media: nil)
+        @color_scheme = validate_symbol(:color_scheme, color_scheme, COLOR_SCHEMES)
+        @reduced_motion = validate_symbol(:reduced_motion, reduced_motion, REDUCED_MOTIONS)
+        @media = validate_symbol(:media, media, MEDIA_TYPES)
+        @viewport = viewport&.transform_keys(&:to_sym)&.freeze
+        @device = device
+        @user_agent = user_agent
+        @locale = locale
+        @timezone = timezone
+        @geolocation = geolocation&.transform_keys(&:to_sym)&.freeze
+        validate_viewport!
+        validate_device!
+        freeze
+      end
+
+      # A copy with +other+'s settings layered on top. Only keys the other side
+      # actually set are overridden, so `emulate(color_scheme: :dark)` on a
+      # session that's already emulating a phone keeps the phone.
+      #
+      # @param other [Emulation, Hash, nil]
+      # @return [Emulation]
+      def merge(other)
+        other = self.class.build(other)
+        self.class.new(**to_h.merge(other.to_h) { |_key, mine, theirs| theirs.nil? ? mine : theirs })
+      end
+
+      # @return [Hash] the knobs, unset ones included as nil
+      def to_h
+        {
+          color_scheme: @color_scheme,
+          reduced_motion: @reduced_motion,
+          viewport: @viewport,
+          device: @device,
+          user_agent: @user_agent,
+          locale: @locale,
+          timezone: @timezone,
+          geolocation: @geolocation,
+          media: @media,
+        }
+      end
+
+      # @return [Boolean] whether every knob is at Chrome's default
+      def default?
+        to_h.values.all?(&:nil?)
+      end
+
+      # @param other [Object]
+      # @return [Boolean] whether the other emulation has the same settings
+      def ==(other)
+        other.is_a?(Emulation) && other.to_h == to_h
+      end
+      alias_method :eql?, :==
+
+      # @return [Integer] a hash of the settings, so an emulation works as a hash key
+      def hash
+        to_h.hash
+      end
+
+      # Push the whole configuration onto a live page. Every knob is sent every
+      # time — including the ones left unset, which are sent as their defaults —
+      # so applying an Emulation always lands the page in exactly this state,
+      # whatever it was in before.
+      #
+      # @param page [Puppeteer::Page]
+      # @return [void]
+      def apply(page)
+        page.viewport = resolved_viewport
+        page.user_agent = resolved_user_agent
+        page.emulate_locale(@locale)
+        page.emulate_timezone(@timezone)
+        apply_geolocation(page)
+        apply_media(page)
+      end
+
+      # The viewport actually sent to Chrome: the device's, with any explicit
+      # `viewport:` keys layered over it, over the plain-desktop default.
+      #
+      # @return [Puppeteer::Viewport]
+      def resolved_viewport
+        base = DEFAULT_VIEWPORT.merge(device_viewport).merge(@viewport || {})
+        Puppeteer::Viewport.new(
+          width: base[:width],
+          height: base[:height],
+          device_scale_factor: base[:scale] || 1,
+          is_mobile: !!base[:mobile],
+          has_touch: !!base[:touch],
+          is_landscape: base[:width] > base[:height],
+        )
+      end
+
+      # @return [String] the user agent sent to Chrome
+      def resolved_user_agent
+        @user_agent || puppeteer_device&.user_agent || DEFAULT_USER_AGENT
+      end
+
+      private
+
+      def apply_geolocation(page)
+        if @geolocation
+          page.geolocation = Puppeteer::Geolocation.new(**@geolocation)
+        else
+          page.client.send_message("Emulation.clearGeolocationOverride")
+        end
+      end
+
+      # Media type and media features share one CDP command, and each omitted
+      # argument resets the other — so they have to go out together rather than
+      # through Puppeteer's two separate setters.
+      #
+      # Unset features are pinned to a value rather than left off. Left off,
+      # Chrome falls back to the *host machine's* appearance, so the same page
+      # would render light on CI and dark on a Mac after sunset. A rendering
+      # library that does that isn't one you can screenshot with.
+      def apply_media(page)
+        page.client.send_message(
+          "Emulation.setEmulatedMedia",
+          media: (@media || :screen).to_s,
+          features: [
+            feature("prefers-color-scheme", @color_scheme || :light),
+            feature("prefers-reduced-motion", @reduced_motion || :no_preference),
+          ],
+        )
+      end
+
+      def feature(name, value)
+        { name: name, value: value.to_s.tr("_", "-") }
+      end
+
+      def device_viewport
+        viewport = puppeteer_device&.viewport
+        return {} unless viewport
+
+        {
+          width: viewport.width,
+          height: viewport.height,
+          scale: viewport.device_scale_factor,
+          mobile: viewport.mobile?,
+          touch: viewport.has_touch?,
+        }
+      end
+
+      def puppeteer_device
+        return nil unless @device
+
+        Puppeteer::DEVICES[@device]
+      end
+
+      def validate_symbol(name, value, allowed)
+        return nil if value.nil?
+
+        symbol = value.to_sym
+        return symbol if allowed.include?(symbol)
+
+        raise ArgumentError, "#{name}: expects one of #{allowed.map(&:inspect).join(", ")}, got #{value.inspect}"
+      end
+
+      def validate_viewport!
+        return if @viewport.nil?
+
+        unknown = @viewport.keys - DEFAULT_VIEWPORT.keys
+        raise ArgumentError, "viewport: unknown key(s) #{unknown.map(&:inspect).join(", ")}" if unknown.any?
+      end
+
+      def validate_device!
+        return if @device.nil? || Puppeteer::DEVICES.key?(@device)
+
+        raise ArgumentError, "device: #{@device.inspect} isn't a device Puppeteer knows about"
+      end
+    end
+  end
+end

data/lib/unmagic/browser/error.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Unmagic
+  class Browser
+    # The base class every failure in this gem inherits from.
+    #
+    # Failures are typed so callers classify them by rescuing, never by parsing
+    # a message — messages carry target URLs and service internals that aren't
+    # a stable contract.
+    #
+    # @example
+    #   begin
+    #     browser.markdown(url)
+    #   rescue Unmagic::Browser::Error::TimedOut
+    #     # the page took too long
+    #   rescue Unmagic::Browser::Error
+    #     # anything else
+    #   end
+    class Error < StandardError
+      # The page, or an element on it, took too long.
+      class TimedOut < Error; end
+
+      # Couldn't reach the page or the rendering service.
+      class Unreachable < Error; end
+
+      # The service answered, but couldn't render the page.
+      class RenderFailed < Error; end
+
+      # A concurrency or rate limit was hit.
+      class RateLimited < Error; end
+
+      # Credentials or configuration are missing.
+      class Misconfigured < Error; end
+
+      # Nothing on the page matched a locator before it gave up. A subclass of
+      # {TimedOut} because locators auto-wait: "not found" always means "still
+      # not there when we ran out of patience".
+      class NotFound < TimedOut; end
+
+      # The session has been closed, so it can't be driven any more.
+      class SessionClosed < Error; end
+    end
+  end
+end

data/lib/unmagic/browser/failures.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "puppeteer"
+require "socket"
+
+module Unmagic
+  class Browser
+    # The one place a Puppeteer, socket or HTTP failure becomes an
+    # {Unmagic::Browser::Error}.
+    #
+    # Callers classify failures by rescuing, so every path out of this gem has
+    # to raise one of our types — including the ones that come out of
+    # puppeteer-ruby, three gems deep, as a bare `Puppeteer::Error`.
+    module Failures
+      # Chrome reports a failed navigation as a `net::ERR_*` protocol message
+      # rather than as a distinct error class, so the message is the only signal
+      # separating "couldn't reach the page" from "the browser broke".
+      UNREACHABLE_PATTERN = /net::ERR_|ERR_NAME_NOT_RESOLVED|ERR_CONNECTION|ERR_ADDRESS|ERR_INTERNET_DISCONNECTED/
+
+      # Anything that means the socket underneath went away.
+      TRANSPORT_ERRORS = [
+        IOError,
+        SocketError,
+        Errno::ECONNRESET,
+        Errno::ECONNREFUSED,
+        Errno::EPIPE,
+        Errno::EHOSTUNREACH,
+        Errno::ENETUNREACH,
+        Errno::ETIMEDOUT,
+      ].freeze
+
+      module_function
+
+      # Run +block+, re-raising anything it throws as a typed error.
+      #
+      # @param doing [String] what was being attempted, for the message
+      # @yield the work to guard
+      # @return [Object] whatever the block returns
+      # @raise [Unmagic::Browser::Error]
+      def wrap(doing)
+        yield
+      rescue Error
+        # Already one of ours — a locator that gave up, say. Leave it alone.
+        raise
+      rescue Puppeteer::TimeoutError => error
+        raise Error::TimedOut, "Timed out #{doing}: #{error.message}"
+      rescue *TRANSPORT_ERRORS => error
+        raise Error::Unreachable, "Couldn't reach the browser while #{doing}: #{error.message}"
+      rescue Puppeteer::Error => error
+        raise Error::Unreachable, "Couldn't reach the page while #{doing}: #{error.message}" if unreachable?(error)
+
+        raise Error::RenderFailed, "The browser failed while #{doing}: #{error.message}"
+      rescue StandardError => error
+        raise Error::RateLimited, "Hit a rate or concurrency limit while #{doing}." if rate_limited?(error)
+
+        raise
+      end
+
+      # @param error [Exception]
+      # @return [Boolean] whether the error is Chrome refusing to load a URL
+      def unreachable?(error)
+        UNREACHABLE_PATTERN.match?(error.message.to_s)
+      end
+
+      # A rejected WebSocket handshake carries its HTTP status on the exception
+      # rather than in the class, and 429 there means a quota — worth retrying
+      # on a cooldown — not a dead network.
+      #
+      # @param error [Exception]
+      # @return [Boolean] whether the error is a quota rejection
+      def rate_limited?(error)
+        error.respond_to?(:response) && error.response.respond_to?(:status) && error.response&.status == 429
+      end
+    end
+  end
+end