websitescreenshotworld 0.1.3

data/lib/websitescreenshotworld/capture_file.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module WebsiteScreenshotWorld
+  # The file a +by_format+ capture answers with, and its metadata. The body is read once: call {#save},
+  # {#read} or {#each_chunk}.
+  #
+  # Inside the block given to {InstantSnap#capture_file} the body streams from the connection. Without a
+  # block it is first buffered to a +Tempfile+, which {#close} deletes.
+  class CaptureFile
+    CHUNK_SIZE = 64 * 1024
+    private_constant :CHUNK_SIZE
+
+    # @return [String, nil] the capture id
+    attr_reader :id
+
+    # @return [String, nil] +delivered+
+    attr_reader :status
+
+    # @return [String, nil] the file's media type, such as +image/png+
+    attr_reader :content_type
+
+    # @return [String, nil] the file name from +Content-Disposition+
+    attr_reader :filename
+
+    # @return [Integer, nil]
+    attr_reader :width
+
+    # @return [Integer, nil]
+    attr_reader :height
+
+    # @return [Integer, nil]
+    attr_reader :size_bytes
+
+    # @return [Integer, nil] how long the capture took
+    attr_reader :duration_ms
+
+    # @return [Integer, nil] the status the website answered with
+    attr_reader :page_http_status
+
+    # @return [String, nil] the default set the capture was taken under
+    attr_reader :spec_version
+
+    # @return [Time, nil]
+    attr_reader :captured_at
+
+    # @return [String, nil] where the browser ended up; absent when that URL is not plain ASCII
+    attr_reader :final_url
+
+    # @return [String, nil] lowercase hex digest of the file
+    attr_reader :sha256
+
+    # @return [Integer, nil] the number of parts, only when the answer has more than one
+    attr_reader :part_count
+
+    # @api private
+    def self.buffered(response)
+      tempfile = Tempfile.new("websitescreenshotworld", binmode: true)
+      Transport.each_chunk(response) { |chunk| tempfile.write(chunk) }
+      tempfile.rewind
+      new(response, tempfile: tempfile)
+    rescue StandardError
+      tempfile&.close!
+      raise
+    end
+
+    # @api private
+    def initialize(response, tempfile: nil)
+      @response = response
+      @tempfile = tempfile
+      @consumed = false
+      read_metadata(response)
+    end
+
+    # Yields the file in chunks as they arrive.
+    #
+    # @yieldparam chunk [String] binary bytes
+    # @return [Enumerator, nil] an enumerator when no block is given
+    # @raise [IOError] when the body has already been read
+    # @raise [ConnectionError] when the connection breaks mid-file
+    def each_chunk(&block)
+      return enum_for(:each_chunk) unless block
+      raise IOError, "the capture file has already been read" if @consumed
+
+      @consumed = true
+      if @tempfile
+        while (chunk = @tempfile.read(CHUNK_SIZE))
+          yield chunk
+        end
+      else
+        Transport.each_chunk(@response, &block)
+      end
+      nil
+    end
+
+    # Reads the whole file into memory.
+    #
+    # @return [String] binary bytes
+    def read
+      buffer = String.new(encoding: Encoding::BINARY)
+      each_chunk { |chunk| buffer << chunk }
+      buffer
+    end
+
+    # Writes the file to a path or an IO without holding it in memory.
+    #
+    # @param target [String, Pathname, #write] a path to create or overwrite, or an open IO
+    # @return [String, Pathname, #write] +target+
+    def save(target)
+      if target.respond_to?(:write)
+        each_chunk { |chunk| target.write(chunk) }
+      else
+        File.open(target, "wb") { |file| each_chunk { |chunk| file.write(chunk) } }
+      end
+      target
+    end
+
+    # @return [String, nil] the path of the buffered copy, when the file was captured without a block
+    def path
+      @tempfile&.path
+    end
+
+    # Deletes the buffered copy, if any.
+    #
+    # @return [void]
+    def close
+      @tempfile&.close!
+    end
+
+    private
+
+    def read_metadata(response)
+      header = ->(name) { response["X-InstantSnap-#{name}"] }
+      @id = header["Id"]
+      @status = header["Status"]
+      @content_type = response["Content-Type"]
+      @filename = self.class.filename_from(response["Content-Disposition"])
+      @width, @height, @size_bytes, @duration_ms, @page_http_status, @part_count =
+        %w[Width Height Size-Bytes Duration-Ms Page-Http-Status Part-Count].map { integer(header[_1]) }
+      @spec_version = header["Spec-Version"]
+      @captured_at = Model.time(header["Captured-At"])
+      @final_url = header["Final-Url"]
+      @sha256 = header["Sha256"]
+    end
+
+    def integer(value)
+      value && Integer(value, 10, exception: false)
+    end
+
+    class << self
+      # @api private
+      def filename_from(disposition)
+        return if disposition.nil?
+
+        name =
+          if (encoded = disposition[/filename\*\s*=\s*UTF-8''([^;\s]+)/i, 1])
+            URI.decode_uri_component(encoded)
+          elsif (quoted = disposition[/filename\s*=\s*"((?:\\.|[^"\\])*)"/i, 1])
+            quoted.gsub(/\\(.)/, '\1')
+          else
+            disposition[/filename\s*=\s*([^;\s]+)/i, 1]
+          end
+        # The name is the server's suggestion; never let it point outside the caller's directory.
+        name && File.basename(name)
+      end
+    end
+  end
+end

data/lib/websitescreenshotworld/catalog.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module WebsiteScreenshotWorld
+  # What the +emulated_device+ and +proxy: { country: }+ options accept, as returned by
+  # {InstantSnap#catalog}.
+  #
+  # @!attribute [r] emulated_devices
+  #   @return [Array<String>]
+  # @!attribute [r] proxy_countries
+  #   @return [Array<String>]
+  Catalog = Data.define(:emulated_devices, :proxy_countries) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(self, hash)
+    end
+  end
+end

data/lib/websitescreenshotworld/client.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module WebsiteScreenshotWorld
+  # One client covers both products. Either key may be left out; calling a product whose key is missing
+  # raises {AuthenticationError} before anything is sent. A client holds no connection and can be shared
+  # between threads.
+  #
+  # @example
+  #   client = WebsiteScreenshotWorld::Client.new
+  #   client.instant_snap.capture(url: "https://example.com")
+  #   client.auto_schedule.trigger_now("7d4e2b1a-6c5f-4a3e-9d8c-2b1a0f9e8d7c")
+  class Client
+    DEFAULT_INSTANT_SNAP_BASE_URL = "https://api.websitescreenshotworld.com"
+    DEFAULT_AUTO_SCHEDULE_BASE_URL = "https://autoapi.websitescreenshotworld.com"
+
+    # @return [InstantSnap]
+    attr_reader :instant_snap
+
+    # @return [AutoSchedule]
+    attr_reader :auto_schedule
+
+    # @param instant_snap_api_key [String, nil] defaults to +ENV["WSW_INSTANT_SNAP_API_KEY"]+
+    # @param auto_schedule_api_key [String, nil] defaults to +ENV["WSW_AUTO_SCHEDULE_API_KEY"]+
+    # @param instant_snap_base_url [String]
+    # @param auto_schedule_base_url [String]
+    # @param capture_timeout [Numeric] seconds a capture or download may take, retries included; the
+    #   server may spend up to 120 seconds on a capture
+    # @param timeout [Numeric] seconds any other call may take, retries included
+    # @param max_retries [Integer] retries after the first attempt for refusals the API marks as
+    #   retryable and for connection failures; 0 turns retrying off
+    def initialize(
+      instant_snap_api_key: ENV.fetch("WSW_INSTANT_SNAP_API_KEY", nil),
+      auto_schedule_api_key: ENV.fetch("WSW_AUTO_SCHEDULE_API_KEY", nil),
+      instant_snap_base_url: DEFAULT_INSTANT_SNAP_BASE_URL,
+      auto_schedule_base_url: DEFAULT_AUTO_SCHEDULE_BASE_URL,
+      capture_timeout: 180,
+      timeout: 30,
+      max_retries: 2
+    )
+      transport = Transport.new(max_retries: max_retries)
+      @instant_snap = InstantSnap.new(transport: transport, base_url: instant_snap_base_url,
+                                      api_key: instant_snap_api_key, timeout: timeout,
+                                      capture_timeout: capture_timeout)
+      @auto_schedule = AutoSchedule.new(transport: transport, base_url: auto_schedule_base_url,
+                                        api_key: auto_schedule_api_key, timeout: timeout)
+    end
+  end
+end

data/lib/websitescreenshotworld/defaults.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module WebsiteScreenshotWorld
+  # A viewport in CSS pixels.
+  #
+  # @!attribute [r] width
+  #   @return [Integer]
+  # @!attribute [r] height
+  #   @return [Integer]
+  # @!attribute [r] device_scale_factor
+  #   @return [Numeric] device pixel ratio
+  Viewport = Data.define(:width, :height, :device_scale_factor) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(self, hash)
+    end
+  end
+
+  # The values a capture takes for options it does not send, and the limits it is checked against,
+  # as returned by {InstantSnap#defaults}. Fields the server leaves out are nil.
+  #
+  # @!attribute [r] spec_version
+  #   @return [String]
+  # @!attribute [r] response_type
+  #   @return [String]
+  # @!attribute [r] media_category
+  #   @return [String]
+  # @!attribute [r] format
+  #   @return [Hash{String => String}, nil] the default format of each media category
+  # @!attribute [r] formats
+  #   @return [Hash{String => Array<String>}, nil] the formats each media category accepts
+  # @!attribute [r] device_category
+  #   @return [String, nil]
+  # @!attribute [r] emulated_device
+  #   @return [String, nil]
+  # @!attribute [r] viewport
+  #   @return [Viewport, nil]
+  # @!attribute [r] zoom
+  #   @return [Integer, nil]
+  # @!attribute [r] scroll_mode
+  #   @return [String, nil]
+  # @!attribute [r] auto_height
+  #   @return [Boolean, nil]
+  # @!attribute [r] auto_height_max_height
+  #   @return [Integer, nil]
+  # @!attribute [r] auto_height_max_height_webp
+  #   @return [Integer, nil]
+  # @!attribute [r] block_ads
+  #   @return [Boolean, nil]
+  # @!attribute [r] block_pop_ups
+  #   @return [Boolean, nil]
+  # @!attribute [r] press_escape
+  #   @return [Boolean, nil]
+  # @!attribute [r] page_timeout_ms
+  #   @return [Integer, nil]
+  # @!attribute [r] scrolling_delay_ms
+  #   @return [Integer, nil]
+  # @!attribute [r] geolocation
+  #   @return [Boolean, nil]
+  # @!attribute [r] geolocation_accuracy
+  #   @return [Numeric, nil]
+  # @!attribute [r] wait_for_cloud_sync
+  #   @return [Boolean, nil]
+  # @!attribute [r] timestamp_watermark
+  #   @return [Hash, nil]
+  # @!attribute [r] page_source
+  #   @return [Hash, nil]
+  # @!attribute [r] limits
+  #   @return [Hash, nil]
+  # @!attribute [r] left_to_the_browser
+  #   @return [Array<String>, nil]
+  Defaults = Data.define(
+    :spec_version, :response_type, :media_category, :format, :formats, :device_category, :emulated_device,
+    :viewport, :zoom, :scroll_mode, :auto_height, :auto_height_max_height, :auto_height_max_height_webp,
+    :block_ads, :block_pop_ups, :press_escape, :page_timeout_ms, :scrolling_delay_ms, :geolocation,
+    :geolocation_accuracy, :wait_for_cloud_sync, :timestamp_watermark, :page_source, :limits, :left_to_the_browser
+  ) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(self, hash, viewport: hash["viewport"]&.then { Viewport.from_api(_1) })
+    end
+  end
+end

data/lib/websitescreenshotworld/error_response.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module WebsiteScreenshotWorld
+  # Turns a refusal into the matching {Error}, following the API's error table.
+  #
+  # @api private
+  module ErrorResponse
+    KINDS = {
+      AuthenticationError => %w[api_key_invalid],
+      InvalidRequestError => %w[
+        validation_failed s3_destination_invalid web_page_id_invalid idempotency_key_reused
+        sitemap_schedule_not_triggerable
+      ],
+      PlanLimitError => %w[quota_exhausted feature_not_available no_trigger_allowance forbidden],
+      NotFoundError => %w[capture_not_found web_page_not_found not_found],
+      RateLimitError => %w[rate_limited queue_full],
+      CaptureError => %w[capture_failed capture_timeout payload_too_large capture_expired],
+      ServiceError => %w[capture_busy service_unavailable internal_error request_cancelled idempotency_key_in_use]
+    }.flat_map { |klass, codes| codes.map { |code| [code, klass] } }.to_h.freeze
+
+    RETRYABLE = %w[rate_limited capture_busy service_unavailable idempotency_key_in_use].freeze
+
+    # A proxy or load balancer answers without the API's JSON body; the code then comes from the status.
+    FALLBACK_BY_STATUS = {
+      400 => "validation_failed",
+      401 => "api_key_invalid",
+      403 => "forbidden",
+      404 => "not_found",
+      409 => "idempotency_key_in_use",
+      413 => "payload_too_large",
+      429 => "rate_limited",
+      500 => "internal_error",
+      502 => "service_unavailable",
+      503 => "service_unavailable",
+      504 => "service_unavailable"
+    }.freeze
+
+    module_function
+
+    # @param response [Net::HTTPResponse]
+    # @return [Error]
+    def build(response)
+      status = response.code.to_i
+      body = response.body
+      refusal = parse(body)
+      code = refusal&.fetch("error_code") || FALLBACK_BY_STATUS.fetch(status, "unknown_error")
+      message = refusal&.dig("error_message") || "The server answered HTTP #{status} without an error body."
+      retry_after = refusal&.dig("retry_after_seconds")
+      retry_after = retry_after_header(response["Retry-After"]) unless retry_after.is_a?(Numeric)
+
+      KINDS.fetch(code, Error).new(
+        message, error_code: code, http_status: status, retry_after_seconds: retry_after, body: body
+      )
+    end
+
+    # @param error_code [String, nil]
+    def retryable?(error_code)
+      RETRYABLE.include?(error_code)
+    end
+
+    def parse(body)
+      refusal = Model.parse_json(body.to_s)
+      refusal if refusal.is_a?(Hash) && refusal["error_code"].is_a?(String)
+    rescue JSON::ParserError
+      nil
+    end
+
+    def retry_after_header(value)
+      return if value.nil?
+
+      Integer(value, exception: false) || [(Time.httpdate(value) - Time.now).ceil, 0].max
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/websitescreenshotworld/errors.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module WebsiteScreenshotWorld
+  # Base class of every error the library raises. Rescue a subclass to handle one kind of refusal, and
+  # switch on {#error_code} rather than on the status or the message.
+  class Error < StandardError
+    # @return [String, nil] the API's stable error token, such as +"rate_limited"+
+    attr_reader :error_code
+
+    # @return [Integer, nil] the HTTP status of the refusal
+    attr_reader :http_status
+
+    # @return [Integer, nil] how long the API asked to wait before retrying
+    attr_reader :retry_after_seconds
+
+    # @return [String, nil] the response body exactly as received
+    attr_reader :body
+
+    # @param message [String]
+    # @param error_code [String, nil]
+    # @param http_status [Integer, nil]
+    # @param retry_after_seconds [Integer, nil]
+    # @param body [String, nil]
+    def initialize(message = nil, error_code: nil, http_status: nil, retry_after_seconds: nil, body: nil)
+      super(message)
+      @error_code = error_code
+      @http_status = http_status
+      @retry_after_seconds = retry_after_seconds
+      @body = body
+    end
+  end
+
+  # The API key is missing, malformed, revoked or belongs to the other product.
+  class AuthenticationError < Error; end
+
+  # The request was refused as invalid, for example a +validation_failed+ or +idempotency_key_reused+.
+  class InvalidRequestError < Error; end
+
+  # The plan has no captures left or does not include the feature.
+  class PlanLimitError < Error; end
+
+  # The capture or web page does not exist for this account.
+  class NotFoundError < Error; end
+
+  # Too many requests. +rate_limited+ is retried automatically before it is raised. +queue_full+ (as many
+  # async captures waiting or running as the organization may have) is not retried; wait
+  # {#retry_after_seconds} before submitting again.
+  class RateLimitError < Error; end
+
+  # The website could not be captured (+capture_failed+, +capture_timeout+, +payload_too_large+), or an async
+  # capture waited too long in the queue (+capture_expired+). Not retried.
+  class CaptureError < Error; end
+
+  # The service is busy or unavailable. The retryable codes are retried automatically before it is raised.
+  class ServiceError < Error; end
+
+  # No answer came back: the connection failed, broke, or the timeout ran out. The original exception is
+  # available as +cause+.
+  class ConnectionError < Error; end
+
+  # A webhook's +X-WSW-Signature+ header is missing, malformed, too old, or does not match the body.
+  class SignatureVerificationError < Error; end
+end

data/lib/websitescreenshotworld/event_data.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module WebsiteScreenshotWorld
+  # One destination of an Auto Schedule delivery, in a +delivery.completed+ event.
+  #
+  # @!attribute [r] slot
+  #   @return [Integer]
+  # @!attribute [r] name
+  #   @return [String]
+  # @!attribute [r] provider
+  #   @return [String]
+  # @!attribute [r] status
+  #   @return [String]
+  # @!attribute [r] completed_at
+  #   @return [Time, nil]
+  Destination = Data.define(:slot, :name, :provider, :status, :completed_at) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(self, hash, completed_at: Model.time(hash["completed_at"]))
+    end
+  end
+
+  # An Auto Schedule capture, the +data+ of its +capture.succeeded+, +capture.failed+,
+  # +delivery.completed+ and +capture.changed+ events.
+  #
+  # @!attribute [r] id
+  #   @return [String]
+  # @!attribute [r] schedule_id
+  #   @return [String] the web page's id, the one {AutoSchedule#trigger_now} takes
+  # @!attribute [r] schedule_name
+  #   @return [String]
+  # @!attribute [r] url
+  #   @return [String]
+  # @!attribute [r] status
+  #   @return [String]
+  # @!attribute [r] media_category
+  #   @return [String]
+  # @!attribute [r] format
+  #   @return [String, nil]
+  # @!attribute [r] page_http_status
+  #   @return [Integer, nil]
+  # @!attribute [r] captured_at
+  #   @return [Time, nil]
+  # @!attribute [r] final_url
+  #   @return [String, nil]
+  # @!attribute [r] media_sha256
+  #   @return [String, nil]
+  # @!attribute [r] page_source
+  #   @return [PageSource, nil] with +format+ and +sha256+ only
+  # @!attribute [r] sitemap_run_id
+  #   @return [String, nil]
+  # @!attribute [r] destinations
+  #   @return [Array<Destination>, nil] on +delivery.completed+ only
+  # @!attribute [r] change_percent
+  #   @return [Numeric, nil] on +capture.changed+ only
+  # @!attribute [r] compared_to_capture_id
+  #   @return [String, nil] on +capture.changed+ only
+  # @!attribute [r] change_image_available
+  #   @return [Boolean, nil] on +capture.changed+ only
+  ScheduledCapture = Data.define(
+    :id, :schedule_id, :schedule_name, :url, :status, :media_category, :format, :page_http_status,
+    :captured_at, :final_url, :media_sha256, :page_source, :sitemap_run_id, :destinations, :change_percent,
+    :compared_to_capture_id, :change_image_available
+  ) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(
+        self, hash,
+        captured_at: Model.time(hash["captured_at"]),
+        page_source: hash["page_source"]&.then { PageSource.from_api(_1) },
+        destinations: hash["destinations"]&.map { Destination.from_api(_1) }
+      )
+    end
+  end
+
+  # The +data+ of a +schedule.paused+ event.
+  #
+  # @!attribute [r] schedule_id
+  #   @return [String]
+  # @!attribute [r] schedule_name
+  #   @return [String]
+  # @!attribute [r] url
+  #   @return [String]
+  # @!attribute [r] reason
+  #   @return [String] such as +quota_exhausted+
+  # @!attribute [r] paused_at
+  #   @return [Time, nil]
+  SchedulePause = Data.define(:schedule_id, :schedule_name, :url, :reason, :paused_at) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(self, hash, paused_at: Model.time(hash["paused_at"]))
+    end
+  end
+
+  # The +data+ of a +sitemap_run.completed+ event.
+  #
+  # @!attribute [r] sitemap_run_id
+  #   @return [String]
+  # @!attribute [r] schedule_id
+  #   @return [String]
+  # @!attribute [r] schedule_name
+  #   @return [String]
+  # @!attribute [r] sitemap_url
+  #   @return [String]
+  # @!attribute [r] status
+  #   @return [String]
+  # @!attribute [r] run_at
+  #   @return [Time, nil]
+  # @!attribute [r] completed_at
+  #   @return [Time, nil]
+  # @!attribute [r] pages_listed
+  #   @return [Integer, nil]
+  # @!attribute [r] pages_matched
+  #   @return [Integer, nil]
+  # @!attribute [r] pages_queued
+  #   @return [Integer, nil]
+  # @!attribute [r] pages_succeeded
+  #   @return [Integer, nil]
+  # @!attribute [r] pages_failed
+  #   @return [Integer, nil]
+  # @!attribute [r] pages_over_limit
+  #   @return [Integer, nil]
+  # @!attribute [r] pages_over_quota
+  #   @return [Integer, nil]
+  # @!attribute [r] message
+  #   @return [String, nil]
+  SitemapRun = Data.define(
+    :sitemap_run_id, :schedule_id, :schedule_name, :sitemap_url, :status, :run_at, :completed_at,
+    :pages_listed, :pages_matched, :pages_queued, :pages_succeeded, :pages_failed, :pages_over_limit,
+    :pages_over_quota, :message
+  ) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(self, hash, run_at: Model.time(hash["run_at"]), completed_at: Model.time(hash["completed_at"]))
+    end
+  end
+
+  # The +data+ of a +ping+ event, a test delivery to a webhook endpoint.
+  #
+  # @!attribute [r] endpoint_id
+  #   @return [String]
+  # @!attribute [r] endpoint_name
+  #   @return [String]
+  # @!attribute [r] message
+  #   @return [String]
+  # @!attribute [r] sent_at
+  #   @return [Time, nil]
+  Ping = Data.define(:endpoint_id, :endpoint_name, :message, :sent_at) do
+    include Model::Raw
+
+    # @api private
+    def self.from_api(hash)
+      Model.build(self, hash, sent_at: Model.time(hash["sent_at"]))
+    end
+  end
+end