wayfarer 0.0.3 → 0.4.0

data/lib/wayfarer/middleware/worker.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    module Worker
+      def self.included(base)
+        base.include(Wayfarer::Redis::Connection)
+        base.include(Wayfarer::Middleware::Stage::API)
+        base.include(InstanceMethods)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def route
+          @route ||= Wayfarer::Routing::RootRoute.new.tap do |route|
+            Docile.dsl_eval(route) { yield if block_given? }
+          end
+        end
+      end
+
+      module InstanceMethods
+        extend Forwardable
+
+        attr_accessor :task
+
+        delegate %i[params adapter] => "task.metadata"
+        delegate %i[browser capybara] => :adapter
+
+        def call(task)
+          self.task = task
+          public_send(task.metadata.action)
+          yield if block_given?
+        end
+
+        def chain
+          Wayfarer::Middleware::Chain.new([*Wayfarer.core_middleware, self])
+        end
+
+        def page(live: false)
+          return task.metadata.page unless live
+
+          task.metadata.page = adapter.live(task.metadata.page)
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/ferrum.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class Ferrum
+      def self.renew_on
+        [::Ferrum::DeadBrowserError]
+      end
+
+      attr_reader :browser
+
+      def initialize
+        @browser = instantiate_browser
+      end
+
+      def fetch(url)
+        browser.goto(url)
+        Result::Success.new(live(nil))
+      end
+
+      def live(_)
+        Wayfarer::Page.new(url: browser.current_url,
+                           body: body,
+                           status_code: browser.network.response.status,
+                           headers: browser.network.response.headers)
+      end
+
+      def body
+        browser.body
+      end
+
+      def renew
+        free
+        @browser = instantiate_browser
+      end
+
+      def free
+        browser&.reset
+        browser&.quit
+        @browser = nil
+        @capybara = nil
+      end
+
+      def capybara
+        @capybara ||= instantiate_capybara_driver
+      end
+
+    private
+
+      def instantiate_browser
+        ::Ferrum::Browser.new(Wayfarer.config.ferrum_options).tap do |browser|
+          browser.headers.set(Wayfarer.config.http_headers)
+        end
+      end
+
+      def instantiate_capybara_driver
+        Capybara.run_server = false
+        Capybara.current_driver = :cuprite
+
+        capybara_driver = Capybara::Cuprite::Driver.new(nil)
+        capybara_driver.instance_variable_set(:@capybara, browser)
+
+        session = Capybara::Session.new(:cuprite, nil)
+        session.instance_variable_set(:@driver, capybara_driver)
+
+        session
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/healer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# An adapter that renews another adapter on certain errors.
+# This wrapper is required because connection_pool assumes
+# all connections are self-healing.
+module Wayfarer
+  module Networking
+    Healer = Struct.new(:adapter) do
+      extend Forwardable
+
+      delegate %i[body browser capybara free] => :adapter
+
+      def fetch(url)
+        adapter.fetch(url)
+      rescue *adapter.class.renew_on => e
+        adapter.renew
+        raise e
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/net_http.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class NetHTTP
+      include Singleton
+
+      def self.renew_on
+        []
+      end
+
+      attr_reader :conn,
+                  :body
+
+      def initialize
+        @conn = Net::HTTP::Persistent.new("wayfarer-#{SecureRandom.uuid}")
+
+        Wayfarer.config.http_headers.each do |key, val|
+          conn.override_headers[key] = val
+        end
+      end
+
+      def renew; end
+
+      def browser; end
+
+      def capybara; end
+
+      def fetch(url)
+        res = conn.request(URI(url))
+
+        return Result::Redirect.new(url, res["location"]) if res.is_a?(Net::HTTPRedirection)
+
+        @body = res.body
+        page = Wayfarer::Page.new(url: url,
+                                  status_code: res.code.to_i,
+                                  body: body,
+                                  headers: res.to_hash)
+
+        Result::Success.new(page)
+      end
+
+      def live(current_page)
+        current_page
+      end
+
+      def free
+        conn.shutdown
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/pool.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class Pool
+      include Singleton
+      extend Forwardable
+
+      attr_reader :pool
+
+      def initialize
+        @pool = ConnectionPool.new(size: Wayfarer.config.adapter_pool_size,
+                                   timeout: Wayfarer.config.adapter_pool_timeout,
+                                   &method(:instantiate_adapter))
+      end
+
+      delegate with: :pool
+
+      def free
+        pool.shutdown(&:free)
+      end
+
+    private
+
+      def instantiate_adapter
+        Healer.new(case Wayfarer.config.adapter
+                   when :net_http then Wayfarer::Networking::NetHTTP.instance
+                   when :selenium then Wayfarer::Networking::Selenium.new
+                   when :ferrum then Wayfarer::Networking::Ferrum.new
+                   end)
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/result.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    # Wrapper structs for page retrieval results.
+    module Result
+      # Signals that a page has been retrieved (2xx or no browser error).
+      # @!attribute [rw] page
+      #   @return [Page] the retrieved page.
+      Success = Struct.new(:page)
+
+      # Signals that a URL resulted in response with 3xx.
+      # @!attribute [rw] redirect_url
+      #   @return [URI] Where to go next.
+      Redirect = Struct.new(:request_url, :redirect_url)
+    end
+  end
+end

data/lib/wayfarer/networking/selenium.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class Selenium
+      def self.renew_on
+        [] # TODO: Figure out when to renew
+      end
+
+      FAKE_STATUS_CODE = 200
+      FAKE_RESPONSE_HEADERS = {}.freeze
+
+      attr_reader :browser
+
+      def initialize
+        @browser = instantiate_browser
+      end
+
+      def fetch(url)
+        browser.navigate.to(url)
+        Result::Success.new(live(nil))
+      end
+
+      def live(_)
+        Wayfarer::Page.new(url: browser.current_url,
+                           body: browser.page_source,
+                           status_code: FAKE_STATUS_CODE,
+                           headers: FAKE_RESPONSE_HEADERS)
+      end
+
+      def body
+        browser.page_source
+      end
+
+      def renew
+        free
+        @browser = instantiate_browser
+      end
+
+      def free
+        browser&.quit
+        @browser = nil
+        @capybara = nil
+      end
+
+      def capybara
+        @capybara ||= instantiate_capybara_driver
+      end
+
+    private
+
+      def instantiate_browser
+        ::Selenium::WebDriver.for(*Wayfarer.config.selenium_argv)
+      end
+
+      def instantiate_capybara_driver
+        Capybara.run_server = false
+        Capybara.current_driver = :selenium
+
+        capybara_driver = Capybara::Selenium::Driver.new(nil)
+        capybara_driver.instance_variable_set(:@capybara, browser)
+
+        session = Capybara::Session.new(:selenium, nil)
+        session.instance_variable_set(:@driver, capybara_driver)
+
+        session
+      end
+    end
+  end
+end

data/lib/wayfarer/page.rb

@@ -1,94 +1,38 @@
 # frozen_string_literal: true
 
-require "ostruct"
-require "forwardable"
-require "mime/types"
-require "mime-types"
-
-require "pismo" unless RUBY_PLATFORM == "java"
-
 module Wayfarer
-  # The representation of fetched pages
   class Page
-    extend Forwardable
-
-    include Finders
+    attr_reader :url,
+                :status_code,
+                :headers
 
-    # @!attribute [r] uri
-    # @return [URI] the URI of the page.
-    attr_reader :uri
-
-    # @!attribute [r] status_code
-    # @return [Fixnum] the response status code.
-    attr_reader :status_code
-
-    # @!attribute [r] body
-    # @return [String] the response body.
     attr_accessor :body
 
-    # @!attribute [r] headers
-    # @return [Hash] the response headers.
-    attr_reader :headers
-
-    def initialize(attrs = {})
-      @uri         = attrs[:uri]
-      @status_code = attrs[:status_code]
-      @body        = attrs[:body]
-      @headers     = attrs[:headers]
+    def initialize(url:, status_code:, headers:, body:)
+      @url = url
+      @status_code = status_code
+      @body = body
+      @headers = headers.transform_keys(&:downcase)
     end
 
-    # Returns a parsed representation of the fetched document depending on the
-    # Content-Type field.
-    # @return [OpenStruct] if the Content-Type field's sub-type is "json".
-    # @return [Nokogiri::XML::Document] if the Content-Type field's sub-type is "xml".
-    # @return [Nokogiri::HTML::Document] otherwise.
     def doc
       return @doc if @doc
 
       # If no Content-Type field is present, assume HTML/XML
-      # TODO: Test
-      unless @headers["content-type"]
-        return @doc = Parsers::XMLParser.parse_html(@body)
-      end
+      return @doc = Wayfarer::Parsing::XML.parse_html(body) unless headers["content-type"]
 
-      content_type = @headers["content-type"].first
+      content_type = headers["content-type"].first
       sub_type = MIME::Types[content_type].first.sub_type
 
-      # TODO: Tests
       @doc = case sub_type
-             when "json"
-               Parsers::JSONParser.parse(@body)
-             when "xml"
-               Parsers::XMLParser.parse_xml(@body)
-             else
-               Parsers::XMLParser.parse_html(@body)
+             when "json" then Wayfarer::Parsing::JSON.parse(body)
+             when "xml" then Wayfarer::Parsing::XML.parse_xml(body)
+             else Wayfarer::Parsing::XML.parse_html(body)
              end
     end
 
-    # Pismo is not supported on JRuby.
-    unless RUBY_PLATFORM == "java"
-      # `#images` is included from the Helpers module
-      # `#body` is an attribute reader defined above
-      delegate (Pismo::Document::ATTRIBUTE_METHODS - %i[images body]) => :pismo
-    end
-
-    private
-
-    # Returns a Pismo document.
-    # @note Not available on JRuby.
-    # @note Only succeeds when {#doc} returns a `Nokogiri::HTML::Document`.
-    # @return [Pismo::Document]
-    def pismo
-      @pismo_doc ||= instantiate_pismo_document
-    end
-
-    def instantiate_pismo_document
-      doc = Pismo::Document.allocate
-      doc.instance_variable_set(:@options, {})
-      doc.instance_variable_set(:@url, uri)
-      doc.instance_variable_set(:@html, body)
-      doc.instance_variable_set(:@doc, self.doc)
-      doc
+    def meta
+      @meta ||= MetaInspector.new(url, document: body)
     end
   end
 end

data/lib/wayfarer/parsing/json.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Parsing
+    module JSON
+    module_function
+
+      def parse(json)
+        if defined?(Oj)
+          Oj.load(json)
+        else
+          ::JSON.parse(json)
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/parsing/xml.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Parsing
+    module XML
+    module_function
+
+      def parse_xml(xml)
+        Nokogiri::XML(xml)
+      end
+
+      def parse_html(html)
+        Nokogiri::HTML(html)
+      end
+    end
+  end
+end

data/lib/wayfarer/redis/barrier.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Redis
+    Barrier = Struct.new(:batch) do
+      include Connection
+
+      def redis_key
+        "wayfarer-barrier-#{batch}"
+      end
+
+      def reset!
+        redis { |conn| conn.del(redis_key) }
+      end
+
+      def seen?(url)
+        !redis { |conn| conn.sadd(redis_key, url) }
+      end
+
+      def peek(urls)
+        major, minor, = Version.determine
+
+        # SMISMEMBER is only supported on Redis >= 6.2.0
+        if major > 6 || (major == 6 && minor >= 2)
+          redis { |conn| conn.smismember(redis_key, urls) }.map { |val| val == 1 }
+        else
+          urls.map { |url| redis { |conn| conn.sismember(redis_key, url) } }
+        end
+      end
+
+      def unsee(url)
+        redis { |conn| conn.srem(redis_key, url) }
+      end
+    end
+  end
+end

data/lib/wayfarer/redis/connection.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Redis
+    module Connection
+      extend Forwardable
+
+      delegate with: "Wayfarer::Redis::Pool.instance"
+
+      alias redis with
+    end
+  end
+end

data/lib/wayfarer/redis/counter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Redis
+    Counter = Struct.new(:batch) do
+      include Connection
+
+      def redis_key
+        "wayfarer-counter-#{batch}"
+      end
+
+      def reset!
+        redis { |conn| conn.del(redis_key) }
+      end
+
+      def value
+        redis { |conn| conn.get(redis_key) }.to_i
+      end
+
+      def increment
+        redis { |conn| conn.incr(redis_key) }
+      end
+
+      def decrement
+        redis { |conn| conn.decr(redis_key) }
+      end
+    end
+  end
+end

data/lib/wayfarer/redis/pool.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Redis
+    class Pool
+      include Singleton
+      extend Forwardable
+
+      attr_reader :pool
+
+      def initialize
+        @pool = ConnectionPool.new(&Wayfarer.config.redis_factory)
+      end
+
+      delegate with: :pool
+    end
+  end
+end

data/lib/wayfarer/redis/version.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Redis
+    module Version
+      extend Connection
+
+    module_function
+
+      def determine
+        @determine ||= server_version.split(".").first(3).map(&:to_i)
+      end
+
+      def server_version
+        redis { |conn| conn.info["redis_version"] }
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/custom_matcher.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class CustomMatcher
+      attr_reader :delegate
+
+      def initialize(delegate = proc)
+        @delegate = delegate
+      end
+
+      def match(url)
+        !!delegate.call(url)
+      end
+
+      def params(_)
+        {}
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/dsl.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    module DSL
+      def url(url, options = {}, &block)
+        add_child_route(URLMatcher.new(url), path_offset, options, &block)
+      end
+
+      def host(host, options = {}, &block)
+        add_child_route(HostMatcher.new(host), path_offset, options, &block)
+      end
+
+      def path(path, options = {}, &block)
+        offset = File.join(path_offset, path)
+        add_child_route(nil, offset, options, &block).tap do |route|
+          route.matcher = PathMatcher.new(offset, route)
+        end
+      end
+
+      def query(fields, options = {}, &block)
+        add_child_route(QueryMatcher.new(fields), path_offset, options, &block)
+      end
+
+      def scheme(scheme, options = {}, &block)
+        add_child_route(SchemeMatcher.new(scheme), path_offset, options, &block)
+      end
+
+      def suffix(suffix, options = {}, &block)
+        add_child_route(SuffixMatcher.new(suffix), path_offset, options, &block)
+      end
+
+      def to(action, options = {}, &block)
+        add_child_route(CustomMatcher.new { true }, path_offset, TargetRoute, options, &block).tap do |route|
+          route.action = action
+        end
+      end
+
+      def custom(delegate, options = {}, &block)
+        add_child_route(CustomMatcher.new(delegate), path_offset, options, &block)
+      end
+
+    private
+
+      # rubocop:disable Style/OptionalArguments
+      def add_child_route(matcher, path_offset, klass = Route, options, &block)
+        klass.new(matcher, path_offset, &block).tap do |route|
+          route.parent = self
+          leaf = options.reduce(route) { |acc, (key, val)| acc.public_send(key, val) }
+          children.push(route)
+          Docile.dsl_eval(leaf, &block) if block_given?
+        end
+      end
+      # rubocop:enable Style/OptionalArguments
+    end
+  end
+end

data/lib/wayfarer/routing/host_matcher.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class HostMatcher
+      attr_reader :host
+
+      def initialize(host)
+        @host = host
+      end
+
+      # rubocop:disable Style/CaseEquality
+      def match(url)
+        @host === url.host
+      end
+      # rubocop:enable Style/CaseEquality
+
+      def params(_)
+        {}
+      end
+    end
+  end
+end