btape 0.1.0 → 0.2.0

data/lib/btape/gif_encoder.rb

@@ -2,62 +2,123 @@
 
 require 'chunky_png'
 require_relative 'lzw_compressor'
+require_relative 'palette'
 
 module Btape
-  # A deliberately small GIF89a encoder. RGB332 gives a fixed 256-colour palette,
-  # avoiding a native image or video dependency.
+  # A deliberately small GIF89a encoder, so that recording needs no native
+  # image or video dependency.
   class GifEncoder
-    def initialize(delay: 10)
-      @delay = delay
+    # A GIF frame delay is two bytes of hundredths of a second.
+    MAX_DELAY = 0xffff
+    # Eight bits per pixel: one index into a 256-colour table.
+    MIN_CODE_SIZE = "\x08"
+
+    def self.for(settings)
+      new(
+        delay: (settings.frame_delay * 100).round,
+        loop_count: settings.loop_count,
+        quantizer: settings.quantizer,
+        scale: settings.scale,
+        width: settings.output_width
+      )
+    end
+
+    def initialize(delay: 10, loop_count: 0, quantizer: :adaptive, scale: 1.0, width: nil, dedupe: true)
+      @delay = delay.clamp(1, MAX_DELAY)
+      @loop_count = loop_count
+      @quantizer = quantizer
+      @scale = scale
+      @width = width
+      @dedupe = dedupe
       @compressor = LzwCompressor.new
     end
 
-    def write(png_paths, output)
-      raise Error, 'no screenshots were captured' if png_paths.empty?
+    # Returns the GIF as a binary String. Frames are PNG paths or ChunkyPNG
+    # canvases. Callers that are not writing to the filesystem — attaching the
+    # GIF to a record, say — want this rather than a file to read back.
+    def encode(frames)
+      raise Error, 'no screenshots were captured' if frames.empty?
 
-      images = png_paths.map { |path| ChunkyPNG::Image.from_file(path) }
+      images = frames.map { |frame| resize(image(frame)) }
       width, height = images.first.dimension.to_a
       raise Error, 'captured screenshots have different dimensions' unless images.all? do |image|
         image.dimension.to_a == [width, height]
       end
 
-      File.binwrite(output, gif(images, width, height))
+      palette = Palette.build(@quantizer, images)
+      gif(collapse(images.map { |image| index(image, palette) }), width, height, palette)
+    end
+
+    # Writes to a path, or to anything that responds to write.
+    def write(frames, output)
+      data = encode(frames)
+      return output.write(data) if output.respond_to?(:write)
+
+      File.binwrite(output, data)
     end
 
     private
 
-    def gif(images, width, height)
+    def image(frame)
+      frame.is_a?(ChunkyPNG::Canvas) ? frame : ChunkyPNG::Image.from_file(frame)
+    end
+
+    def resize(image)
+      target = @width || (image.width * @scale).round
+      return image if target == image.width || target < 1
+
+      # A wide enough source rounds its scaled height down to nothing, and a
+      # zero-height canvas becomes a GIF no decoder can show.
+      height = [(image.height * target.to_f / image.width).round, 1].max
+      image.resample_bilinear(target, height)
+    end
+
+    def index(image, palette)
+      image.pixels.map { |pixel| palette.index_for(pixel) }
+    end
+
+    # A run that sits on one page for a second is a dozen identical frames.
+    # Holding the first for longer says the same thing in a fraction of the
+    # bytes, and spares a decoder the redundant frames.
+    def collapse(frames)
+      frames.each_with_object([]) do |pixels, collapsed|
+        previous = collapsed.last
+        if @dedupe && previous && previous.first == pixels
+          previous[1] = [previous[1] + @delay, MAX_DELAY].min
+        else
+          collapsed << [pixels, @delay]
+        end
+      end
+    end
+
+    def gif(frames, width, height, palette)
       data = +'GIF89a'.b
-      data << [width, height, 0b11110111, 0, 0].pack('vvCCC') << palette
-      data << "!\xFF\x0BNETSCAPE2.0\x03\x01\x00\x00\x00".b
-      images.each { |image| write_frame(data, image, width, height) }
+      data << [width, height, 0b11110111, 0, 0].pack('vvCCC') << palette.to_gct
+      data << netscape
+      frames.each { |pixels, delay| write_frame(data, pixels, delay, width, height) }
       data << ';'.b
     end
 
-    def write_frame(data, image, width, height)
-      data << "!\xF9\x04\x00".b << [@delay].pack('v') << "\x00\x00".b
-      data << ','.b << [0, 0, width, height, 0].pack('vvvvC')
-      compressed = lzw(image)
-      data << "\x08".b
-      compressed.bytes.each_slice(255) { |slice| data << slice.length.chr << slice.pack('C*') }
-      data << "\x00".b
+    # The application extension that carries the loop count. Zero loops
+    # forever, which is what an unattended recording usually wants.
+    def netscape
+      +"!\xFF\x0BNETSCAPE2.0\x03\x01".b << [@loop_count].pack('v') << "\x00".b
     end
 
-    def palette
-      (0..255).map { |i| [((i >> 5) & 7) * 255 / 7, ((i >> 2) & 7) * 255 / 7, (i & 3) * 255 / 3].pack('C3') }.join.b
+    def write_frame(data, pixels, delay, width, height)
+      data << "!\xF9\x04\x00".b << [delay].pack('v') << "\x00\x00".b
+      data << ','.b << [0, 0, width, height, 0].pack('vvvvC')
+      data << MIN_CODE_SIZE.b
+      lzw(pixels).bytes.each_slice(255) { |slice| data << slice.length.chr << slice.pack('C*') }
+      data << "\x00".b
     end
 
-    def lzw(image)
-      pixels = image.pixels.map { |pixel| rgb332(pixel) }
+    def lzw(pixels)
       codes = []
       @compressor.call(pixels) { |code, width| codes << [code, width] }
       pack_codes(codes)
     end
 
-    def rgb332(pixel)
-      (ChunkyPNG::Color.r(pixel) & 0xe0) | ((ChunkyPNG::Color.g(pixel) & 0xe0) >> 3) | (ChunkyPNG::Color.b(pixel) >> 6)
-    end
-
     def pack_codes(codes)
       buffer = 0
       count = 0

data/lib/btape/null_logger.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Btape
+  # Stands in when no logger was given, so the rest of the code can log
+  # unconditionally. Only the three levels btape uses are defined, and any
+  # Logger — Rails' included — can be passed instead.
+  class NullLogger
+    def debug(*); end
+
+    def info(*); end
+
+    def warn(*); end
+  end
+end

data/lib/btape/palette.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require 'chunky_png'
+
+module Btape
+  # The 256 colours a GIF frame is written in, and the mapping from a pixel to
+  # one of them.
+  #
+  # Fixed spreads its colours evenly across the whole RGB cube regardless of
+  # what is being encoded, which costs nothing to build but wastes most of the
+  # palette on colours the image does not contain. Adaptive chooses the
+  # colours from the image itself, which is what keeps text edges and
+  # gradients from banding.
+  module Palette
+    SIZE = 256
+    # Colours are bucketed to five bits per channel before anything looks at
+    # them. It collapses a screenshot's million pixels into a few thousand
+    # distinct entries, and the error it introduces is below what the 256
+    # colours of a GIF can express anyway.
+    LEVELS = 32
+    CELLS = LEVELS**3
+
+    module_function
+
+    def build(quantizer, images)
+      quantizer.to_sym == :adaptive ? Adaptive.from(images) : Fixed.new
+    end
+
+    def cell(pixel)
+      ((ChunkyPNG::Color.r(pixel) >> 3) << 10) |
+        ((ChunkyPNG::Color.g(pixel) >> 3) << 5) |
+        (ChunkyPNG::Color.b(pixel) >> 3)
+    end
+
+    def rgb(cell)
+      [((cell >> 10) & 31) * 255 / 31, ((cell >> 5) & 31) * 255 / 31, (cell & 31) * 255 / 31]
+    end
+
+    # Pads a list of colours out to a full global colour table.
+    def colour_table(colours)
+      colours.map { |colour| colour.pack('C3') }.join.b.ljust(SIZE * 3, "\x00".b)
+    end
+
+    # Three bits of red, three of green, two of blue. Cheap, and independent
+    # of the image, but only eight distinct levels of red to spend on a
+    # gradient that may need far more.
+    class Fixed
+      def index_for(pixel)
+        (ChunkyPNG::Color.r(pixel) & 0xe0) |
+          ((ChunkyPNG::Color.g(pixel) & 0xe0) >> 3) |
+          (ChunkyPNG::Color.b(pixel) >> 6)
+      end
+
+      def to_gct
+        Palette.colour_table(
+          (0...SIZE).map { |i| [((i >> 5) & 7) * 255 / 7, ((i >> 2) & 7) * 255 / 7, (i & 3) * 255 / 3] }
+        )
+      end
+    end
+
+    # Colours chosen from the frames themselves by median cut.
+    class Adaptive
+      # Enough of each frame to describe what colours it uses. Reading every
+      # pixel of every frame would cost far more and change the answer very
+      # little.
+      MAX_SAMPLES_PER_IMAGE = 60_000
+
+      def self.from(images)
+        new(histogram(images))
+      end
+
+      def self.histogram(images)
+        counts = Hash.new(0)
+        images.each do |image|
+          pixels = image.pixels
+          stride = [pixels.length / MAX_SAMPLES_PER_IMAGE, 1].max
+          index = 0
+          while index < pixels.length
+            counts[Palette.cell(pixels[index])] += 1
+            index += stride
+          end
+        end
+        counts
+      end
+
+      def initialize(histogram)
+        @colours = MedianCut.new(histogram).colours
+        # One entry per five-bit colour cell, filled in as cells are met. A
+        # screenshot touches a small fraction of them, so searching the
+        # palette for the rest would be work thrown away.
+        @lookup = Array.new(CELLS)
+      end
+
+      attr_reader :colours
+
+      def index_for(pixel)
+        cell = Palette.cell(pixel)
+        @lookup[cell] || (@lookup[cell] = nearest(cell))
+      end
+
+      def to_gct
+        Palette.colour_table(@colours)
+      end
+
+      private
+
+      def nearest(cell)
+        red, green, blue = Palette.rgb(cell)
+        best = 0
+        shortest = nil
+        @colours.each_with_index do |(r, g, b), index|
+          distance = ((red - r)**2) + ((green - g)**2) + ((blue - b)**2)
+          next unless shortest.nil? || distance < shortest
+
+          shortest = distance
+          best = index
+        end
+        best
+      end
+    end
+
+    # Repeatedly splits the colours in use along their widest axis, so that
+    # each of the 256 boxes it ends up with holds a similar share of the
+    # pixels. Colours a frame leans on get more of the palette than colours it
+    # barely touches.
+    class MedianCut
+      # Colours in a box, as [red, green, blue, weight] entries. The stats a
+      # split needs are worked out once per box: they are read every round,
+      # and the box they describe never changes.
+      class Box
+        def initialize(entries)
+          @entries = entries
+          @bounds = compute_bounds
+          @axis = (0..2).max_by { |channel| @bounds[channel].last - @bounds[channel].first }
+          @weight = entries.sum(&:last)
+        end
+
+        attr_reader :entries, :axis, :weight
+
+        def splittable?
+          @entries.length > 1
+        end
+
+        # Both how far the box spreads and how many pixels are in it: a wide
+        # box nothing is using does not deserve the palette entry.
+        def priority
+          (@bounds[@axis].last - @bounds[@axis].first) * @weight
+        end
+
+        def split
+          sorted = @entries.sort_by { |entry| entry[@axis] }
+          [Box.new(sorted.shift(median_index(sorted))), Box.new(sorted)]
+        end
+
+        def colour
+          (0..2).map { |channel| @entries.sum { |entry| entry[channel] * entry.last } / @weight }
+        end
+
+        private
+
+        def compute_bounds
+          (0..2).map { |channel| @entries.map { |entry| entry[channel] }.minmax }
+        end
+
+        # Splits where half the weight lies, leaving at least one entry each
+        # side however lopsided the weights are.
+        def median_index(sorted)
+          half = @weight / 2.0
+          running = 0
+          taken = sorted.take_while do |entry|
+            running += entry.last
+            running < half
+          end.length
+          taken.clamp(1, sorted.length - 1)
+        end
+      end
+
+      def initialize(histogram, size: SIZE)
+        @entries = histogram.map { |cell, weight| Palette.rgb(cell) << weight }
+        @size = size
+      end
+
+      def colours
+        return [[0, 0, 0]] if @entries.empty?
+
+        boxes = [Box.new(@entries)]
+        while boxes.length < @size
+          index = widest(boxes)
+          break unless index
+
+          boxes[index, 1] = boxes[index].split
+        end
+        boxes.map(&:colour)
+      end
+
+      private
+
+      def widest(boxes)
+        boxes.each_index.select { |index| boxes[index].splittable? }.max_by { |index| boxes[index].priority }
+      end
+    end
+  end
+end

data/lib/btape/parser.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require 'shellwords'
+require_relative 'duration'
+require_relative 'recorder'
+require_relative 'settings'
 
 module Btape
   Command = Struct.new(:name, :arguments, :line_number, keyword_init: true)
@@ -8,7 +11,13 @@ module Btape
   # Parses .tape scripts into a list of Command structs, raising ScriptError
   # for unknown commands, wrong argument counts, or invalid argument values.
   class Parser
-    ARITY = { 'Output' => 1, 'Viewport' => 1, 'Goto' => 1, 'Click' => 1, 'Type' => 2, 'Sleep' => 1 }.freeze
+    # An arity is either an exact count or a range, for commands whose last
+    # argument is optional.
+    ARITY = {
+      'Output' => 1, 'Viewport' => 1, 'Goto' => 1, 'Click' => 1, 'Type' => 2, 'Sleep' => 1,
+      'Set' => 2, 'Screenshot' => 0..1, 'Evaluate' => 1, 'WaitFor' => 1..2, 'WaitForJS' => 1..2,
+      'Frame' => 1, 'Press' => 1..2
+    }.freeze
 
     def parse(source)
       source.each_line.with_index(1).filter_map do |line, number|
@@ -29,7 +38,7 @@ module Btape
       raise ScriptError.new(number, "unknown command #{name.inspect}") unless ARITY.key?(name)
 
       arity = ARITY.fetch(name)
-      unless words.length == arity
+      unless arity === words.length # rubocop:disable Style/CaseEquality
         raise ScriptError.new(number, "#{name} expects #{arity} argument(s), got #{words.length}")
       end
 
@@ -41,9 +50,35 @@ module Btape
       case name
       when 'Viewport' then validate_viewport(arguments.first, number)
       when 'Sleep' then validate_sleep(arguments.first, number)
+      when 'Set' then Settings.validate!(*arguments)
+      when 'Screenshot' then validate_frame_name(arguments.first, number)
+      when 'WaitFor', 'WaitForJS' then validate_wait_timeout(name, arguments, number)
+      when 'Press' then validate_press_count(arguments[1], number)
       end
     end
 
+    def validate_press_count(value, number)
+      return if value.nil? || (/\A\d+\z/.match?(value) && value.to_i.positive?)
+
+      raise ScriptError.new(number, 'Press count must be a positive integer')
+    end
+
+    def validate_wait_timeout(name, arguments, number)
+      timeout = arguments[1]
+      return if timeout.nil? || Duration.valid?(timeout)
+
+      raise ScriptError.new(number, "#{name} timeout #{Duration::DESCRIPTION}")
+    end
+
+    # The name becomes part of a filename, so keep it to something that
+    # cannot escape the frames directory. Recorder rejects the same names, but
+    # catching it here names the line the script has to fix.
+    def validate_frame_name(value, number)
+      return if value.nil? || Recorder::FRAME_NAME.match?(value)
+
+      raise ScriptError.new(number, 'Screenshot name must be letters, numbers, dashes, dots or underscores')
+    end
+
     def validate_viewport(value, number)
       match = /\A(\d+)x(\d+)\z/.match(value)
       valid = match&.captures&.all? { |part| part.to_i.positive? }
@@ -51,9 +86,9 @@ module Btape
     end
 
     def validate_sleep(value, number)
-      return if /\A(\d+(?:\.\d+)?)(ms|s)\z/.match?(value)
+      return if Duration.valid?(value)
 
-      raise ScriptError.new(number, 'Sleep duration must use ms or s (for example, 500ms or 1.5s)')
+      raise ScriptError.new(number, "Sleep duration #{Duration::DESCRIPTION}")
     end
   end
 end

data/lib/btape/recorder.rb

@@ -1,20 +1,41 @@
 # frozen_string_literal: true
 
+require 'monitor'
+
 module Btape
-  # Captures periodic screenshots of a page on a background thread until
-  # stopped, building the frame sequence GifEncoder turns into a GIF.
+  # Captures the frames a GIF is built from.
+  #
+  # In :interval mode it screenshots the page on a background thread until
+  # stopped. In :manual mode it captures only when the script asks with a
+  # Screenshot command, which is what a tape wants when it needs one frame per
+  # page rather than a few hundred near-identical ones.
   class Recorder
-    def initialize(page, directory, interval: 0.1)
+    # A name becomes part of a filename, so it may not carry anything that
+    # would put the frame somewhere other than the frames directory. Parser
+    # checks it too, to report a bad Screenshot line with its line number;
+    # this is the check for everyone else, since capture is public API.
+    FRAME_NAME = /\A[\w.-]+\z/
+
+    def initialize(page, directory, interval: 0.1, mode: :interval, on_frame: nil, max_frames: nil,
+                   lock: Monitor.new)
       @page = page
       @directory = directory
       @interval = interval
+      @mode = mode.to_sym
+      @on_frame = on_frame
+      @max_frames = max_frames
+      # Shared with the executor, so a screenshot and a command are never in
+      # flight against the same page at once.
+      @lock = lock
       @paths = []
-      @mutex = Mutex.new
+      @named_paths = {}
     end
 
-    attr_reader :paths
+    attr_reader :paths, :named_paths
 
     def start
+      return if manual?
+
       capture
       @running = true
       @thread = Thread.new do
@@ -37,14 +58,46 @@ module Btape
       raise @error if @error
     end
 
-    private
+    # Captures one frame now. A name puts it at a predictable path, so a
+    # caller can pick a particular frame out of the run.
+    def capture(name: nil)
+      validate_name!(name)
+      path, index = @lock.synchronize do
+        exhausted! if @max_frames && @paths.length >= @max_frames
 
-    def capture
-      @mutex.synchronize do
-        path = File.join(@directory, format('frame-%06d.png', @paths.length))
+        path = File.join(@directory, filename(name))
         screenshot(path)
+        index = @paths.length
         @paths << path
+        @named_paths[name] = path if name
+        [path, index]
       end
+      # Outside the lock: the callback is the caller's code and may take as
+      # long as it likes without holding up the next capture.
+      @on_frame&.call(path, index)
+      path
+    end
+
+    private
+
+    # A page that hangs would otherwise be screenshotted until the disk ran
+    # out, which on a shared host takes more than this run down with it.
+    def exhausted!
+      raise Error, "stopped after #{@max_frames} frames; raise Set MaxFrames to record for longer"
+    end
+
+    def manual?
+      @mode == :manual
+    end
+
+    def validate_name!(name)
+      return if name.nil? || FRAME_NAME.match?(name)
+
+      raise Error, "frame name must be letters, numbers, dashes, dots or underscores: #{name.inspect}"
+    end
+
+    def filename(name)
+      name ? "frame-#{name}.png" : format('frame-%06d.png', @paths.length)
     end
 
     def screenshot(path)

data/lib/btape/result.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Btape
+  # What a run produced. `output_path` is where the GIF was written, or nil
+  # when the caller passed an IO for it to be written into instead.
+  #
+  # Callers that want the frames themselves (to attach a still somewhere, say)
+  # pass `frames_directory:` to Runner#run and read `frame_paths`.
+  #
+  # Without a `frames_directory:` the frames live in a temporary directory
+  # that is removed as the run unwinds, so `frame_paths` and `named_frames`
+  # are empty rather than lists of paths that no longer exist. `frame_count`
+  # is always the number of frames that went into the GIF.
+  #
+  # `named_frames` maps the name a `Screenshot NAME` command gave a frame to
+  # its path, for picking one particular frame out of a run.
+  Result = Struct.new(:output_path, :frame_paths, :named_frames, :frame_count, :width, :height, keyword_init: true)
+end