wayfarer 0.0.3 → 0.4.3

data/lib/wayfarer/config/struct.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    class Struct
+      module ClassMethods
+        attr_accessor :members
+      end
+
+      module InstanceMethods
+        extend Forwardable
+
+        delegate members: "self.class"
+
+        attr_reader :env
+
+        def initialize(env = ENV)
+          @env = env
+
+          define_writers
+          define_readers
+        end
+
+      private
+
+        def define_writers
+          members.each { |key, _| define_writer(key) }
+        end
+
+        def define_writer(key)
+          define_singleton_method(:"#{key}=") do |val|
+            set(key, val)
+          end
+        end
+
+        def define_readers
+          members.each { |key, options| define_reader(key, **options) }
+        end
+
+        def define_reader(key, env_key: nil, type: nil, default: nil)
+          define_singleton_method(key.to_sym) do
+            get(key) || set(key, env_val(env_key, type) || default)
+          end
+        end
+
+        def env_val(env_key, type)
+          return nil unless env_key
+          return nil unless env.key?(env_key)
+
+          Strconv.parse(env[env_key], type)
+        end
+
+        def get(key)
+          instance_variable_get(:"@#{key}")
+        end
+
+        def set(key, val)
+          instance_variable_set(:"@#{key}", val)
+        end
+      end
+
+      def self.new(members)
+        Class.new do
+          include InstanceMethods
+          extend ClassMethods
+
+          self.members = members
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/gc.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  GC = Struct.new(:job) do
+    def run
+      task = job.task
+
+      return unless task.counter.decrement <= 0
+
+      task.barrier.reset!
+      task.counter.reset!
+      job.run_callbacks(:batch)
+    end
+  end
+end

data/lib/wayfarer/middleware/chain.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    Chain = Struct.new(:middlewares) do
+      def call(task)
+        traverse = lambda do |i|
+          if middlewares[i]
+            middlewares[i].call(task) { traverse.call(i + 1) }
+          elsif block_given?
+            yield
+          end
+        end
+
+        traverse.call(0)
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/dedup.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Dedup
+      def call(task)
+        return if task.barrier.seen?(task.url)
+
+        begin
+          yield if block_given?
+        rescue StandardError => e
+          task.barrier.unsee(task.url)
+          raise e
+        end
+
+        staged_urls = task.metadata.staged_urls
+        return if staged_urls.none?
+
+        inclusion = task.barrier.peek(staged_urls.to_a)
+        unseen = staged_urls.zip(inclusion).reject { |_, seen| seen }.map(&:first)
+        task.metadata.staged_urls = SortedSet.new(unseen)
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/fetch.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Fetch
+      module API
+        def agent
+          task.metadata.agent
+        end
+
+        def page(live: false)
+          return task.metadata.page unless live
+
+          task.metadata.page = agent.live&.page || task.metadata.page
+        end
+      end
+
+      include Wayfarer::Middleware::Stage::API
+
+      attr_reader :pool
+      attr_accessor :task
+
+      def initialize
+        @pool = Wayfarer::Networking::Pool.instance
+      end
+
+      def call(task)
+        self.task = task
+
+        pool.with do |context|
+          result = task.job.run_callbacks :fetch do
+            context.fetch(task.url)
+          end
+
+          case result
+          when Networking::Result::Redirect
+            stage(result.redirect_url)
+          when Networking::Result::Success
+            task.metadata.agent = context.instance
+            task.metadata.page = result.page
+            yield if block_given?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/normalize.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Normalize
+      def call(task)
+        yield if block_given?
+
+        task.metadata.staged_urls = SortedSet.new(normalized_urls(task).compact)
+      end
+
+    private
+
+      def normalized_urls(task)
+        task.metadata.staged_urls.map(&method(:normalize))
+      end
+
+      def normalize(url)
+        NormalizeUrl.process(url)
+      rescue NormalizeUrl::InvalidURIError
+        nil
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/router.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Router
+      module API
+        def self.included(base)
+          base.include(InstanceMethods)
+          base.extend(ClassMethods)
+        end
+
+        module InstanceMethods
+          def steer
+            []
+          end
+
+          def params
+            task.metadata.params
+          end
+        end
+
+        module ClassMethods
+          def router
+            @router ||= Wayfarer::Routing::Router.new
+          end
+
+          def route(&block)
+            router.draw(&block) if block_given?
+          end
+
+          def steer(&block)
+            define_method(:steer) { block.call(task) }
+          end
+        end
+      end
+
+      def call(task)
+        router = task.job.class.router
+        url = Addressable::URI.parse(task.url)
+
+        case result = router.invoke(url, task.job.steer)
+        when Routing::Result::Mismatch
+          return
+        when Routing::Result::Match
+          task.metadata.action = result.action
+          task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new(result.params)
+        end
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/stage.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Stage
+      module API
+        def stage(urls)
+          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url) }
+        end
+      end
+
+      def call(task)
+        task.metadata.staged_urls = SortedSet.new
+
+        yield if block_given?
+
+        task.metadata.staged_urls.each do |url|
+          task.job.class.crawl_later(url, batch: task.batch)
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/worker.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    module Worker
+      def self.included(base)
+        base.include(Wayfarer::Redis::Connection)
+        base.include(Wayfarer::Middleware::Stage::API)
+        base.include(Wayfarer::Middleware::Router::API)
+        base.include(Wayfarer::Middleware::Fetch::API)
+        base.include(Wayfarer::Callbacks)
+        base.include(InstanceMethods)
+      end
+
+      module InstanceMethods
+        def call(task)
+          run_callbacks :action do
+            public_send(task.metadata.action)
+          end
+
+          yield if block_given?
+        end
+
+        def chain
+          Wayfarer::Middleware::Chain.new([*Wayfarer.middleware, self])
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/capybara.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class Capybara
+      include Strategy
+
+      def create
+        ::Capybara::Session.new(Wayfarer.config.capybara.driver, nil)
+      end
+
+      def destroy(instance)
+        instance.quit
+      end
+
+      def navigate(instance, url)
+        instance.visit(url)
+      end
+
+      def live(instance)
+        success(url: instance.current_url,
+                body: instance.html,
+                status_code: instance.status_code,
+                headers: instance.response_headers)
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/context.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    Context = Struct.new(:strategy) do
+      def fetch(url)
+        supervise { strategy.fetch(instance, url) }
+      end
+
+      def live
+        supervise { strategy.live(instance) }
+      end
+
+      def renew
+        strategy.destroy(instance)
+      ensure
+        @instance = nil
+      end
+
+      def instance
+        @instance ||= strategy.create
+      end
+
+    private
+
+      def supervise
+        yield
+      rescue *strategy.renew_on => e
+        renew
+      ensure
+        # If renewing raises, re-raise the originally caught exception
+        raise e if e
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/ferrum.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class Ferrum
+      include Strategy
+
+      def renew_on
+        [::Ferrum::DeadBrowserError]
+      end
+
+      def create
+        ::Ferrum::Browser.new(Wayfarer.config.ferrum.options).tap do |browser|
+          browser.headers.set(Wayfarer.config.network.http_headers)
+        end
+      end
+
+      def destroy(instance)
+        instance.reset
+        instance.quit
+      end
+
+      def navigate(instance, url)
+        instance.goto(url)
+      end
+
+      def live(instance)
+        success(url: instance.current_url,
+                body: instance.body,
+                status_code: instance.network.response.status,
+                headers: instance.network.response.headers)
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/http.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class HTTP
+      include Strategy
+
+      CONNECTION_NAME = "wayfarer"
+
+      def create
+        Net::HTTP::Persistent.new(name: CONNECTION_NAME).tap do |conn|
+          Wayfarer.config.network.http_headers.each do |key, val|
+            conn.override_headers[key] = val
+          end
+        end
+      end
+
+      def destroy(instance)
+        instance.shutdown
+      end
+
+      def fetch(instance, url)
+        res = instance.request(URI(url))
+
+        return redirect(res["location"]) if res.is_a?(Net::HTTPRedirection)
+
+        success(url: url,
+                status_code: res.code.to_i,
+                body: res.body,
+                headers: res.to_hash.transform_values(&:first))
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/pool.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class Pool
+      include Singleton
+
+      cattr_accessor :registry, default: { http: HTTP,
+                                           ferrum: Ferrum,
+                                           selenium: Selenium,
+                                           capybara: Capybara }
+
+      def pool
+        @pool ||= ConnectionPool.new(size: Wayfarer.config.network.pool_size,
+                                     timeout: Wayfarer.config.network.pool_timeout,
+                                     &method(:context))
+      end
+
+      def with(&block)
+        pool.with(&block)
+      rescue ConnectionPool::TimeoutError => e
+        raise Wayfarer::UserAgentTimeoutError, e
+      end
+
+      def free
+        pool.shutdown(&:renew)
+      end
+
+    private
+
+      def context
+        Wayfarer::Networking::Context.new(strategy)
+      end
+
+      def strategy
+        self.class.registry[Wayfarer.config.network.agent].new
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/result.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    # Wrapper structs for page retrieval results.
+    module Result
+      # Signals that a page has been retrieved (2xx or no browser error).
+      # @!attribute [rw] page
+      #   @return [Page] the retrieved page.
+      Success = Struct.new(:page)
+
+      # Signals that a URL resulted in response with 3xx.
+      # @!attribute [rw] redirect_url
+      #   @return [URI] Where to go next.
+      Redirect = Struct.new(:redirect_url)
+    end
+  end
+end

data/lib/wayfarer/networking/selenium.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    class Selenium
+      include Strategy
+
+      MOCK_STATUS_CODE = 200
+      MOCK_RESPONSE_HEADERS = {}.freeze
+
+      def create
+        ::Selenium::WebDriver.for(Wayfarer.config.selenium.driver, **options)
+      end
+
+      def destroy(instance)
+        instance.quit
+      end
+
+      def navigate(instance, url)
+        instance.navigate.to(url)
+      end
+
+      def live(instance)
+        success(url: instance.current_url,
+                body: instance.page_source,
+                status_code: MOCK_STATUS_CODE,
+                headers: MOCK_RESPONSE_HEADERS)
+      end
+
+    private
+
+      def options
+        Wayfarer.config.selenium.options.merge(http_client: http_client)
+      end
+
+      def http_client
+        ::Selenium::WebDriver::Remote::Http::Default.new.tap do |client|
+          client.read_timeout = Wayfarer.config.selenium.client_timeout
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/networking/strategy.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Networking
+    module Strategy
+      def renew_on
+        []
+      end
+
+      def fetch(instance, url)
+        navigate(instance, url)
+        live(instance)
+      end
+
+      def navigate(_instance, _url)
+        raise NoMethodError
+      end
+
+      def live(_instance); end
+
+      def create
+        raise NoMethodError
+      end
+
+      def destroy(_instance); end
+
+    private
+
+      def success(...)
+        Wayfarer::Networking::Result::Success.new(Wayfarer::Page.new(...))
+      end
+
+      def redirect(...)
+        Wayfarer::Networking::Result::Redirect.new(...)
+      end
+    end
+  end
+end