unmagic-browser 0.1.0

data/lib/unmagic/browser/console/renderer.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+begin
+  require "rouge"
+rescue LoadError
+  # Syntax highlighting is a nicety for a human at a terminal, not something a
+  # gem that renders web pages should make anyone install. Without it the
+  # console still works — the source just comes out plain.
+  nil
+end
+
+module Unmagic
+  class Browser
+    module Console
+      # Turns whatever the last expression evaluated to into something worth
+      # looking at.
+      #
+      # IRB normally echoes `inspect`, which for this gem means a screenful of
+      # escaped HTML or, worse, a megabyte of PNG bytes. Every result comes
+      # through here instead: source gets highlighted, images get drawn, and
+      # everything else falls back to a highlighted `inspect`.
+      module Renderer
+        # Long enough to read a page's shape, short enough not to lose the
+        # prompt. The value itself is untouched — `puts _` still gives you all
+        # of it.
+        MAX_LINES = 60
+
+        # A belt to MAX_LINES' braces: one minified line of HTML is a screenful
+        # on its own.
+        MAX_CHARS = 8_000
+
+        class << self
+          # The Rouge theme used for highlighting, when Rouge is available.
+          #
+          # @return [Class]
+          attr_writer :theme
+
+          # @return [Class] defaults to Monokai
+          def theme
+            @theme ||= Rouge::Themes::Monokai if highlighting?
+          end
+
+          # @return [Boolean] whether Rouge is loaded and can highlight
+          def highlighting?
+            defined?(Rouge) ? true : false
+          end
+
+          # @param value [Object] whatever the last expression returned
+          # @return [String] what to print
+          def render(value)
+            case value
+            when Image then image(value)
+            when Document then document(value)
+            when Code then code(value)
+            when Unmagic::Browser::Session then session(value)
+            when Unmagic::Browser::Element then element(value)
+            when Unmagic::Browser then browser(value)
+            when Unmagic::Browser::Emulation then emulation(value)
+            when Array then array(value)
+            when String then string(value)
+            when nil then Terminal.dim("nil")
+            else ruby(value.inspect)
+            end
+          end
+
+          # Drawing happens as a side effect, because the image goes out as a
+          # raw escape sequence rather than as text IRB could indent or wrap.
+          #
+          # @param value [Image]
+          # @return [String] the caption printed under the image
+          def image(value)
+            path = Graphics.show(value)
+            width, height = value.dimensions
+            size = width ? "#{width}×#{height}" : "#{value.bytesize} bytes"
+            caption = Terminal.dim("#{size} · #{path}")
+            Terminal.graphics? ? caption : "#{Terminal.paint(Terminal::YELLOW, "[image]")} #{caption}"
+          end
+
+          # @param value [Document]
+          # @return [String] where it was written
+          def document(value)
+            path = Graphics.save(value)
+            "#{Terminal.paint(Terminal::YELLOW, "[#{value.extension}]")} " +
+              Terminal.dim("#{value.bytesize} bytes · #{path}")
+          end
+
+          # @param value [Code]
+          # @return [String] the source, highlighted and truncated
+          def code(value)
+            body, omitted = clamp(value)
+            [highlight(body, value.language), footer(value, omitted)].compact.join("\n")
+          end
+
+          # @param value [String]
+          # @return [String] the string, as prose if it's long, as Ruby if it's short
+          def string(value)
+            return ruby(value.inspect) unless value.include?("\n") || value.length > 100
+
+            body, omitted = clamp(value)
+            [body, footer(value, omitted)].compact.join("\n")
+          end
+
+          # @param value [Unmagic::Browser::Session]
+          # @return [String] where the session is, and what it's emulating
+          def session(value)
+            return Terminal.paint(Terminal::GREY, "session (closed)") if value.closed?
+
+            # Plenty of pages have no <title> — Hacker News' login page, for one
+            # — and a bold empty string reads as something having gone wrong.
+            title = value.title.to_s.strip
+            heading = title.empty? ? Terminal.dim("(untitled)") : Terminal.paint(Terminal::BOLD, title)
+            lines = ["#{Terminal.paint(Terminal::GREEN, "●")} #{heading}"]
+            lines << Terminal.dim("  #{value.current_url}")
+            knobs = value.emulation.to_h.compact
+            lines << Terminal.dim("  emulating #{describe(knobs)}") if knobs.any?
+            lines.join("\n")
+          end
+
+          # @param value [Unmagic::Browser::Element]
+          # @return [String] the element's tag and the text in it
+          def element(value)
+            text = value.text
+            text = "#{text[0, 80]}…" if text.length > 80
+            tag = Terminal.paint(Terminal::BLUE, "<#{value.tag_name}>")
+            text.empty? ? tag : "#{tag} #{text.inspect}"
+          end
+
+          # @param value [Array]
+          # @return [String] a numbered list for elements, highlighted Ruby otherwise
+          def array(value)
+            return ruby(value.inspect) unless value.any? && value.all?(Unmagic::Browser::Element)
+
+            width = value.size.to_s.length
+            value.each_with_index.map do |item, index|
+              "#{Terminal.dim("[#{index.to_s.rjust(width)}]")} #{element(item)}"
+            end.join("\n")
+          end
+
+          # @param value [Unmagic::Browser]
+          # @return [String] the driver it's running against
+          def browser(value)
+            "#{Terminal.paint(Terminal::BOLD, "browser")} #{Terminal.dim("via #{value.driver.name}")}"
+          end
+
+          # @param value [Unmagic::Browser::Emulation]
+          # @return [String] the knobs that are set, or that none are
+          def emulation(value)
+            knobs = value.to_h.compact
+            return Terminal.dim("emulating nothing — desktop, light, screen") if knobs.empty?
+
+            knobs.map { |key, setting| "#{Terminal.paint(Terminal::BLUE, key.to_s)} #{setting.inspect}" }.join("\n")
+          end
+
+          # @param source [String] Ruby source, usually from #inspect
+          # @return [String] it, highlighted
+          def ruby(source)
+            highlight(source, "ruby")
+          end
+
+          # @param source [String]
+          # @param language [String] a Rouge lexer name
+          # @return [String] the source with terminal colour, or untouched
+          #   without a tty or without Rouge
+          def highlight(source, language)
+            return source unless Terminal.tty? && highlighting?
+
+            lexer = Rouge::Lexer.find(language) || Rouge::Lexers::PlainText
+            formatter.format(lexer.new.lex(source)).chomp
+          rescue StandardError
+            source # Never let a highlighting failure swallow the value itself.
+          end
+
+          # Every terminal that can draw an image can also do 24-bit colour, so
+          # there's no reason to quantise to 256.
+          #
+          # @return [Rouge::Formatter]
+          def formatter
+            @formatter ||= Rouge::Formatters::TerminalTruecolor.new(theme.new)
+          end
+
+          private
+
+          def describe(knobs)
+            knobs.map { |key, setting| "#{key}: #{setting.inspect}" }.join(", ")
+          end
+
+          # @return [Array(String, Integer)] as much as we'll print, and how many
+          #   lines were held back
+          def clamp(value)
+            lines = value.lines
+            kept = lines.first(MAX_LINES)
+            kept = kept.join[0, MAX_CHARS].lines if kept.join.length > MAX_CHARS
+            [kept.join.chomp, lines.size - kept.size]
+          end
+
+          def footer(value, omitted)
+            return nil unless omitted.positive?
+
+            Terminal.dim("… #{omitted} more lines (#{value.length} chars in all — `puts _` for everything)")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unmagic/browser/console/terminal.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "io/console"
+
+module Unmagic
+  class Browser
+    module Console
+      # What this terminal can do, and how to draw on it.
+      module Terminal
+        # The palette, kept to the 256-colour cube so it looks the same wherever
+        # the console runs. Only {Renderer}'s syntax highlighting reaches for
+        # true colour, and only when Rouge is there to do it.
+
+        # Back to the terminal's own colours.
+        RESET = "\e[0m"
+
+        # For anything secondary — paths, sizes, counts.
+        DIM = "\e[2m"
+
+        # For the one thing on the line that matters.
+        BOLD = "\e[1m"
+
+        # For a session that's gone.
+        GREY = "\e[38;5;245m"
+
+        # For a session that's live, and for headings.
+        GREEN = "\e[38;5;114m"
+
+        # For elements, keys, and the verbs in the banner.
+        BLUE = "\e[38;5;75m"
+
+        # For what the terminal can't draw and had to write to a file instead.
+        YELLOW = "\e[38;5;179m"
+
+        # `TIOCGWINSZ` is the only way to ask a terminal how big a character cell
+        # is in *pixels*, which is what an image has to be measured against. The
+        # request number is baked into each kernel's headers rather than
+        # standardised.
+        WINSIZE_REQUEST = {
+          /darwin|bsd/ => 0x40087468,
+          /linux/ => 0x5413,
+        }.find { |pattern, _| pattern.match?(RUBY_PLATFORM) }&.last
+
+        # A sane guess for a cell when the terminal won't say.
+        FALLBACK_CELL = [8, 17].freeze
+
+        module_function
+
+        # Terminals that implement the kitty graphics protocol. Ghostty and
+        # WezTerm both do, and neither says "kitty" anywhere in `$TERM`.
+        #
+        # @return [Boolean] whether an image can be drawn inline
+        def graphics?
+          return false unless tty?
+          return true if ENV["KITTY_WINDOW_ID"] || ENV["WEZTERM_PANE"]
+
+          "#{ENV["TERM"]} #{ENV["TERM_PROGRAM"]}".downcase.match?(/kitty|ghostty|wezterm/)
+        end
+
+        # @return [Boolean] whether output is going to a terminal at all
+        def tty?
+          $stdout.tty?
+        end
+
+        # @return [Integer] the terminal's width in characters
+        def columns
+          IO.console&.winsize&.last || 80
+        rescue StandardError
+          80
+        end
+
+        # @return [Array(Integer, Integer)] the width and height of one character cell, in pixels
+        def cell
+          rows, cols, width, height = window
+          return FALLBACK_CELL if width.nil? || width.zero? || cols.zero? || rows.zero?
+
+          [width / cols, height / rows]
+        end
+
+        # @param text [String]
+        # @return [String] the text, dimmed, if the terminal has colour
+        def dim(text)
+          paint(DIM, text)
+        end
+
+        # @param colour [String] one of the escape constants above
+        # @param text [String]
+        # @return [String] the text in that colour, if the terminal has colour
+        def paint(colour, text)
+          tty? ? "#{colour}#{text}#{RESET}" : text.to_s
+        end
+
+        # @return [Array(Integer, Integer, Integer, Integer)] rows, columns, and
+        #   the window's width and height in pixels; zeroes when the terminal
+        #   won't say
+        def window
+          return [0, 0, 0, 0] unless WINSIZE_REQUEST && tty?
+
+          buffer = [0, 0, 0, 0].pack("S_4")
+          $stdout.ioctl(WINSIZE_REQUEST, buffer)
+          buffer.unpack("S_4")
+        rescue StandardError
+          [0, 0, 0, 0]
+        end
+      end
+    end
+  end
+end

data/lib/unmagic/browser/console/values.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Unmagic
+  class Browser
+    module Console
+      # A String that also knows how it wants to be shown.
+      #
+      # Sniffing a value to guess whether it's HTML, Markdown or PNG bytes would
+      # be guesswork the console doesn't need to do: the helper that produced it
+      # knows exactly what it asked for. These carry that answer through to the
+      # renderer while staying plain Strings for everything else, so
+      # `html.include?("...")` and `puts markdown` work the way you'd expect.
+      class Tagged < String
+        # @param value [String] the bytes themselves
+        # @param name [String] what to call it in the console, and on disk
+        def initialize(value, name:)
+          super(value.to_s)
+          @name = name
+        end
+
+        # @return [String]
+        attr_reader :name
+      end
+
+      # Source the renderer should syntax-highlight.
+      class Code < Tagged
+        # @param value [String] the source
+        # @param language [String] a Rouge lexer name, e.g. `"html"`
+        # @param name [String] what to call it in the console
+        def initialize(value, language:, name: language)
+          super(value, name: name)
+          @language = language
+        end
+
+        # @return [String] the Rouge lexer to highlight with
+        attr_reader :language
+      end
+
+      # Image bytes the renderer should draw in the terminal.
+      class Image < Tagged
+        # @param value [String] PNG bytes
+        # @param name [String] a filename stem for the copy written to disk
+        def initialize(value, name: "screenshot")
+          super(value, name: name)
+        end
+
+        # The image's pixel size, read straight out of the PNG header — the IHDR
+        # chunk is always the first one, at a fixed offset.
+        #
+        # @return [Array(Integer, Integer), nil] width and height, or nil if it isn't a PNG
+        def dimensions
+          return nil unless start_with?("\x89PNG\r\n\x1A\n".b)
+
+          byteslice(16, 8)&.unpack("N2")
+        end
+
+        # @return [String] the file extension for the copy written to disk
+        def extension
+          "png"
+        end
+      end
+
+      # Bytes there's no way to draw in a terminal — a PDF. Saved and pointed at.
+      class Document < Tagged
+        # @param value [String] the bytes
+        # @param extension [String] the file extension to save under
+        # @param name [String] a filename stem for the copy written to disk
+        def initialize(value, extension: "pdf", name: "document")
+          super(value, name: name)
+          @extension = extension
+        end
+
+        # @return [String] the file extension for the copy written to disk
+        attr_reader :extension
+      end
+    end
+  end
+end

data/lib/unmagic/browser/console.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require_relative "../browser"
+require_relative "console/values"
+require_relative "console/terminal"
+require_relative "console/graphics"
+require_relative "console/renderer"
+require_relative "console/prompt"
+require_relative "console/banner"
+require_relative "console/helpers"
+
+module Unmagic
+  class Browser
+    # A browser you can type at.
+    #
+    #     open "https://example.com"
+    #     click "More information"
+    #     screenshot
+    #
+    # Not required by `unmagic/browser` — it's for a person at a terminal, not
+    # for an app, and it takes over `main` and IRB's output when it starts.
+    # Require it yourself:
+    #
+    #     require "unmagic/browser/console"
+    #     Unmagic::Browser::Console.start
+    #
+    # Every result is rendered rather than inspected: source is syntax
+    # highlighted, screenshots are drawn in the terminal, elements and sessions
+    # get a summary worth reading. Highlighting wants
+    # [rouge](https://github.com/rouge-ruby/rouge) — the console works without
+    # it, just in monochrome — and drawing images wants a terminal that speaks
+    # the [kitty graphics
+    # protocol](https://sw.kovidgoyal.net/kitty/graphics-protocol/) (kitty,
+    # Ghostty, WezTerm). Neither is a dependency.
+    module Console
+      # Raised when a verb needs a page and there isn't one open.
+      class NothingOpen < Error; end
+
+      # One line of a dotenv file: an optional `export`, a name, an `=`, a value
+      # that may be quoted, and an optional trailing comment.
+      ENV_LINE = /\A\s*(?:export\s+)?([A-Z_][A-Z0-9_]*)\s*=\s*(.*?)\s*\z/i
+
+      class << self
+        # The browser every verb runs against, built on first use.
+        #
+        # @return [Unmagic::Browser]
+        def browser
+          @browser ||= Unmagic::Browser.new(driver: @driver || :local)
+        end
+
+        # @return [Unmagic::Browser::Session, nil] the session the verbs are driving
+        attr_accessor :session
+
+        # Throw away the browser and any session, so the next verb builds a new one.
+        #
+        # @param driver [Symbol, Unmagic::Browser::Driver::Base, nil] what to build next time
+        # @return [void]
+        def reset(driver = nil)
+          session&.close
+          @session = nil
+          @browser&.close
+          @browser = nil
+          @driver = driver if driver
+        end
+
+        # Mix the verbs into an object — `main`, in a console — and start IRB.
+        #
+        # @param context [Binding] whose receiver gets the verbs, and where IRB runs
+        # @param banner [Boolean] print the banner and the verb list first
+        # @param prompt [Symbol] `:plain` for `browser>`, or `:page` for a prompt
+        #   that follows the browser around — see {Prompt}
+        # @return [void]
+        def start(context: TOPLEVEL_BINDING, banner: true, prompt: :plain)
+          require "irb"
+          require "irb/workspace"
+
+          # Only the one object gets the verbs, rather than every Object in the
+          # process — `open` and `select` shadow Kernel's, which is a trade
+          # worth making for `main` and nothing else.
+          context.receiver.extend(Helpers)
+          at_exit { reset rescue nil }
+
+          if banner
+            puts(Banner.render)
+            puts(Banner.help)
+          end
+
+          ARGV.clear
+          IRB.setup(nil)
+          # After `setup`, never before: it resets IRB.conf, which would throw
+          # away a prompt registered ahead of it and leave PROMPT_MODE pointing
+          # at nothing.
+          configure
+          irb = IRB::Irb.new(IRB::WorkSpace.new(context))
+          Prompt.install(irb.context) if prompt == :page
+          irb.run(IRB.conf)
+        end
+
+        # Read a dotenv file into ENV, if it's there. Values already in the
+        # environment win — an export in your shell should beat a checked-in
+        # file.
+        #
+        # @param path [String] the file to read
+        # @return [Array<String>] the names that were set
+        def load_env(path)
+          return [] unless File.exist?(path)
+
+          File.readlines(path).filter_map do |line|
+            next if line.strip.empty? || line.strip.start_with?("#")
+
+            name, raw = ENV_LINE.match(line)&.captures
+            next unless name && !ENV.key?(name)
+
+            ENV[name] = unquote(raw)
+            name
+          end
+        end
+
+        private
+
+        # IRB echoes `inspect` by default, which for a page of HTML is a
+        # screenful of escapes and for a screenshot is a megabyte of PNG bytes.
+        # Every result goes through the renderer instead.
+        def configure
+          IRB::Inspector.def_inspector([:unmagic_browser]) do |value, colorize: true|
+            Renderer.render(value)
+          end
+
+          # What the prompt is unless `start` was asked for `prompt: :page`, in
+          # which case {Prompt} takes these over on the context once IRB is
+          # built.
+          IRB.conf[:PROMPT][:UNMAGIC_BROWSER] = {
+            PROMPT_I: "browser> ",
+            PROMPT_S: "browser%l ",
+            PROMPT_C: "browser* ",
+            # No `=> `: the renderer draws images and highlighted source, and a
+            # prefix in front of a picture reads as a mistake.
+            RETURN: "%s\n",
+          }
+
+          IRB.conf[:INSPECT_MODE] = :unmagic_browser
+          IRB.conf[:PROMPT_MODE] = :UNMAGIC_BROWSER
+        end
+
+        # Strip one layer of matching quotes and any trailing comment. A `#`
+        # inside quotes stays — that's what quoting it is for — which is why the
+        # comment has to come off after the quotes rather than before.
+        def unquote(raw)
+          case raw
+          when /\A"((?:[^"\\]|\\.)*)"(?:\s+#.*)?\z/m
+            Regexp.last_match(1).gsub('\\n', "\n").gsub('\\"', '"').gsub("\\\\", "\\")
+          when /\A'([^']*)'(?:\s+#.*)?\z/m
+            Regexp.last_match(1)
+          else
+            raw.sub(/\s+#.*\z/, "")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unmagic/browser/driver/base.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Unmagic
+  class Browser
+    module Driver
+      # What every driver has to be able to do, and the parts none of them need
+      # to write twice.
+      #
+      # A driver answers two questions: how do I get a browser, and what's the
+      # cheapest way to grab a single page? Subclasses answer the first with
+      # {#connect_browser} and {#disconnect_browser}. The second is answered
+      # here, for free, by opening a throwaway page on a shared connection —
+      # {Cloudflare} overrides it because it has a REST API that needs no
+      # browser at all.
+      class Base
+        # Seconds a one-off page grab waits for the page to settle.
+        DEFAULT_TIMEOUT = 30
+
+        # When a page counts as loaded. `networkidle2` waits out the XHR that a
+        # JS-rendered page fires after `load`, which is the whole reason to run
+        # a real browser instead of fetching HTML.
+        WAIT_UNTIL = "networkidle2"
+
+        def initialize
+          @scratch_mutex = Mutex.new
+          @scratch = nil
+        end
+
+        # Open a browser of this driver's kind, for a caller who'll hold it.
+        #
+        # @return [Connection]
+        def open_connection
+          Failures.wrap("connecting to the browser") { Connection.new(self, connect_browser) }
+        end
+
+        # @param url [String] the page to render
+        # @return [String] the page as Markdown
+        def markdown(url)
+          Unmagic::Browser::Markdown.from_html(html(url))
+        end
+
+        # @param url [String] the page to render
+        # @return [String] the page's HTML, after JavaScript has run
+        def html(url)
+          on_scratch_page(url) { |page| page.content }
+        end
+
+        # @param url [String] the page to render
+        # @param full_page [Boolean] capture the whole scrollable page
+        # @param selector [String, nil] capture just the first element matching this CSS
+        # @return [String] PNG bytes
+        # @raise [Error::NotFound] when +selector+ matches nothing
+        def screenshot(url, full_page: false, selector: nil)
+          on_scratch_page(url) do |page|
+            if selector
+              element = page.query_selector(selector)
+              raise Error::NotFound, "No element matches #{selector.inspect} on #{url}." unless element
+
+              element.screenshot(type: "png")
+            else
+              page.screenshot(type: "png", full_page: full_page)
+            end
+          end
+        end
+
+        # @param url [String] the page to render
+        # @return [String] PDF bytes
+        def pdf(url)
+          on_scratch_page(url) { |page| page.pdf }
+        end
+
+        # Drop the shared connection one-off grabs run on. Sessions are
+        # unaffected — each holds its own browser.
+        #
+        # @return [void]
+        def close
+          scratch = nil
+          @scratch_mutex.synchronize do
+            scratch = @scratch
+            @scratch = nil
+          end
+          scratch&.close
+        end
+
+        # @return [String] a short description used in log lines
+        def name
+          self.class.name.split("::").last.downcase
+        end
+
+        protected
+
+        # @return [Puppeteer::Browser] a browser of this driver's kind
+        # @raise [NotImplementedError] when a subclass hasn't implemented it
+        def connect_browser
+          raise NotImplementedError, "#{self.class} must implement #connect_browser"
+        end
+
+        # @param browser [Puppeteer::Browser] the browser to shut down
+        # @return [void]
+        # @raise [NotImplementedError] when a subclass hasn't implemented it
+        def disconnect_browser(browser)
+          raise NotImplementedError, "#{self.class} must implement #disconnect_browser"
+        end
+
+        # Render +url+ on a throwaway page and hand it to the block. The page
+        # goes away afterwards; the browser under it is reused by the next
+        # one-off, because launching Chrome per call would dwarf the work.
+        #
+        # @param url [String] the page to render
+        # @yieldparam page [Puppeteer::Page]
+        # @return [Object] whatever the block returns
+        def on_scratch_page(url)
+          Failures.wrap("rendering #{url}") do
+            page = scratch.new_page
+            begin
+              Emulation.new.apply(page)
+              page.goto(url, wait_until: WAIT_UNTIL, timeout: DEFAULT_TIMEOUT * 1000)
+              yield page
+            ensure
+              begin
+                page&.close
+              rescue StandardError
+                nil # A page whose browser already went away can't be closed, and needn't be.
+              end
+            end
+          end
+        end
+
+        private
+
+        # The connection one-off grabs share. Created on first use and closed at
+        # exit, so a `browser.markdown(url)` in a script doesn't leave a Chrome
+        # behind.
+        def scratch
+          @scratch_mutex.synchronize do
+            if @scratch.nil? || @scratch.closed?
+              @scratch = Connection.new(self, connect_browser)
+              Driver.close_at_exit(self)
+            end
+            @scratch
+          end
+        end
+      end
+    end
+  end
+end