wayfarer 0.0.3 → 0.4.0

data/lib/wayfarer/http_adapters/adapter_pool.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-require "forwardable"
-require "connection_pool"
-
-module Wayfarer
-  module HTTPAdapters
-    # A connection pool that hands out HTTP adapters.
-    # @private
-    class AdapterPool
-      extend Forwardable
-
-      def initialize(job)
-        @job = job
-        @config = job.config
-
-        size = @config.connection_count
-        timeout = @config.connection_timeout
-
-        @pool = ConnectionPool.new(
-          size: size,
-          timeout: timeout,
-          &method(:instantiate_adapter)
-        )
-      end
-
-      # Shuts down all HTTP adapters
-      def free
-        @pool.shutdown(&:free)
-      end
-
-      private
-
-      def instantiate_adapter
-        adapter = if @config.http_adapter == :selenium
-                    HTTPAdapters::SeleniumAdapter.new(@config)
-                  else
-                    HTTPAdapters::NetHTTPAdapter.instance(@config)
-                  end
-
-        @job.run_hook(
-          :setup_adapter,
-          adapter,
-          adapter.try(:driver),
-          adapter.try(:browser)
-        )
-
-        adapter
-      end
-
-      def method_missing(method, *argv, &proc)
-        super if method == :shutdown
-        @pool.public_send(method, *argv, &proc)
-      end
-
-      def respond_to_missing?(method, private = false)
-        return false if method == :shutdown
-        @pool.respond_to?(method) || super
-      end
-    end
-  end
-end

data/lib/wayfarer/http_adapters/net_http_adapter.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-require "securerandom"
-require "net/http"
-require "net/http/persistent"
-
-module Wayfarer
-  module HTTPAdapters
-    # A singleton adapter for net-http-persistent.
-    # @api private
-    class NetHTTPAdapter
-      # Supported standard lib classes
-      RECOGNIZED_URI_TYPES = [
-        URI::HTTP,
-        URI::HTTPS
-      ].freeze
-
-      MalformedURI = Class.new(StandardError)
-      MalformedRedirectURI = Class.new(StandardError)
-      MaximumRedirectCountReached = Class.new(StandardError)
-
-      attr_accessor :request_header_overrides
-
-      # TODO: Remove default parameter value
-      def self.instance(config = Wayfarer.config)
-        @@instance ||= new(config)
-      end
-
-      def initialize(config)
-        @config = config
-        @conn = Net::HTTP::Persistent.new("wayfarer-#{SecureRandom.uuid}")
-      end
-
-      # This is a singleton class. Use ::instance instead.
-      private_class_method :new
-
-      # Fetches a page.
-      # @return [Page]
-      # @raise [MalformedURI] if the URI is not supported.
-      # @raise [MalformedRedirectURI] if a redirection URI is not supported.
-      # @raise [MaximumRedirectCountReached] if too many redirections are
-      # encountered.
-      def fetch(uri, redirects_followed = 0)
-        if !RECOGNIZED_URI_TYPES.include?(uri.class)
-          raise _ = if redirects_followed.positive?
-                      MalformedRedirectURI
-                    else
-                      MalformedURI
-                    end
-        elsif redirects_followed > @config.max_http_redirects
-          raise MaximumRedirectCountReached
-        end
-
-        res = @conn.request(uri)
-
-        if res.is_a? Net::HTTPRedirection
-          redirect_uri = URI(res["location"])
-          return fetch(redirect_uri, redirects_followed + 1)
-        end
-
-        Page.new(
-          uri: uri,
-          status_code: res.code.to_i,
-          body: res.body,
-          headers: res.to_hash
-        )
-      rescue SocketError
-        raise MalformedURI
-      end
-
-      # Shuts down all connections.
-      def free
-        @conn.shutdown
-      end
-    end
-  end
-end

data/lib/wayfarer/http_adapters/selenium_adapter.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-
-require "selenium-webdriver"
-require "selenium/emulated_features"
-require "capybara"
-
-module Wayfarer
-  module HTTPAdapters
-    # An adapter for Selenium WebDrivers
-    # @api private
-    class SeleniumAdapter
-      # @!attribute [r] driver
-      # @return [URI] the Selenium WebDriver.
-      attr_reader :driver
-
-      def initialize(config = Wayfarer.config)
-        @config = config
-      end
-
-      # Fetches a page.
-      # @return [Page]
-      def fetch(uri)
-        driver.navigate.to(uri)
-
-        Page.new(
-          uri: @driver.current_url,
-          status_code: @driver.response_code,
-          body: @driver.page_source,
-          headers: @driver.response_headers
-        )
-      end
-
-      # Closes the driver.
-      def reload!
-        @driver&.close
-        @driver = nil
-      end
-
-      # Quits the browser.
-      def free
-        @driver&.quit
-        @driver = nil
-      end
-
-      # The WebDriver.
-      def driver
-        @driver ||= instantiate_driver
-      end
-
-      # A Capybara driver that wraps the {#driver}.
-      # @see https://github.com/teamcapybara/capybara Capybara
-      def browser
-        @browser ||= instantiate_capybara_driver
-      end
-
-      private
-
-      def instantiate_driver
-        driver = Selenium::WebDriver.for(*@config.selenium_argv)
-        driver.manage.window.size = Selenium::WebDriver::Dimension.new(
-          *@config.window_size
-        )
-        driver
-      end
-
-      def instantiate_capybara_driver
-        Capybara.run_server = false
-        Capybara.current_driver = :selenium
-
-        capybara_driver = Capybara::Selenium::Driver.new(nil)
-        capybara_driver.instance_variable_set(:@browser, driver)
-
-        session = Capybara::Session.new(:selenium, nil)
-        session.instance_variable_set(:@driver, capybara_driver)
-
-        session
-      end
-    end
-  end
-end

data/lib/wayfarer/job.rb

@@ -1,211 +0,0 @@
-# frozen_string_literal: true
-
-require "forwardable"
-require "hooks"
-require "active_job"
-
-# TODO: I only want deep_dup
-require "active_support/all"
-
-module Wayfarer
-  # A {Job} is a class that has a {Routing::Router} with many {Routing::Rule}s
-  # which are matched against a URI. Rules map URIs onto job instance methods.
-  # Under the hood, jobs are instantiated within separate threads by a
-  # {Processor}. Every instance gets its own thread. If a URI is matched, its
-  # {Page} is retrieved, and made available to instance methods via {#page}.
-  #
-  # Jobs implement ActiveJob's Job API and are therefore compatible with a wide
-  # range of job queues. To run a job immediately, call ::perform_now.
-  # enqueue a job, call ::perform_later.
-  #
-  # @see https://github.com/rails/rails/tree/master/activejob rails/activejob
-  # @see http://edgeguides.rubyonrails.org/active_job_basics.html ActiveJob Basics
-  class Job < ActiveJob::Base
-    extend Forwardable
-
-    include Hooks
-    include Locals
-
-    # @!group Callbacks
-
-    # Callback that fires __once__ before any pages are retrieved.
-    # @method before_crawl
-    # @scope class
-    define_hook :before_crawl
-
-    # Callback that fires __once__ after all pages have been retrieved and
-    # processing is done.
-    # @method after_crawl
-    # @scope class
-    define_hook :after_crawl
-
-    # Callback that fires when HTTP adapters are instantiated.
-    # @method setup_adapter
-    # @scope class
-    # @yield [[HTTPAdapters::NetHTTPAdapter, HTTPAdapters::SeleniumAdapter], [Selenium::WebDriver::Driver, nil], [Capybara::Selenium::Driver, nil]]
-    define_hooks :setup_adapter
-
-    # @!endgroup
-
-    class << self
-      extend Forwardable
-
-      # @!attribute [w] router
-      attr_writer :router
-
-      # @!attribute [w] config
-      attr_writer :config
-
-      # Returns a class copy.
-      def prepare
-        duplicate = dup
-        duplicate.router = router.dup
-        duplicate.locals = locals.deep_dup
-        duplicate.config = config.dup
-
-        duplicate.locals.each do |(key, val)|
-          duplicate.locals[key] = Locals.thread_safe_counterpart(val)
-        end
-
-        duplicate.locals.each do |(key, _)|
-          duplicate.send(:define_method, key) do duplicate.locals[key] end
-          duplicate.send(:define_singleton_method, key) do
-            duplicate.locals[key]
-          end
-        end
-
-        duplicate
-      end
-
-      # A configuration based off the global {Wayfarer.config}.
-      # @yield [Configuration]
-      # @return [Configuration]
-      def config
-        @config ||= Wayfarer.config.clone
-        yield(@config) if block_given?
-        @config
-      end
-
-      # A router.
-      # If a block is passed in, it is evaluated within the {Router}'s instance.
-      # @return [Routing::Router]
-      def router(&proc)
-        @router ||= Routing::Router.new
-        @router.instance_eval(&proc) if block_given?
-        @router
-      end
-
-      alias route router
-      alias routes router
-
-      # Overshadows ActiveJob::Base's own logger
-      delegate logger: :config
-    end
-
-    # @!attribute [r] staged_uris
-    # @return [Array<String>, Array<URI>] URIs to stage for the next cycle.
-    # @see #stage
-    attr_reader :staged_uris
-
-    # @!attribute [rw] page
-    attr_writer :page
-
-    # @!attribute [rw] adapter
-    attr_accessor :adapter
-
-    # @!attribute [rw] params
-    attr_accessor :params
-
-    def initialize(*argv)
-      @halts = false
-      @staged_uris = []
-      super(*argv)
-    end
-
-    # Whether this job will stop processing.
-    def halts?
-      @halts
-    end
-
-    # Performs this job.
-    # @note ActiveJob API
-    # @override
-    def perform(*uris)
-      Crawl.new(self.class, *uris).execute
-    end
-
-    protected
-
-    # All following instance methods are available within actions.
-
-    # Sets a halting flag that signals the processor to stop its work.
-    def halt
-      @halts = true
-    end
-
-    # Adds URIs to process in the next cycle.
-    # If a relative path is given, an absolute URI is constructed from the
-    # current {#page}'s URI.
-    # @param [String, URI, Array<String>, Array<URI>]
-    def stage(*uris)
-      expanded = uris.flatten.map do |u|
-        if (uri = URI(u)).absolute?
-          uri
-        else
-          # URI#join would discard the path of page.uri.path
-          current = page.uri.dup
-          current.path = File.join(page.uri.path, uri.path)
-          current
-        end
-      end
-
-      # This method has somewhat become the guard keeper for invalid URIs that
-      # would lead to exceptions otherwise down the line
-      supported = expanded.select do |uri|
-        HTTPAdapters::NetHTTPAdapter::RECOGNIZED_URI_TYPES.any? do |type|
-          uri.is_a?(type)
-        end
-      end
-
-      @staged_uris.push(*supported)
-    end
-
-    # The {Page} representing the URI currently processed by an action.
-    # When using the Selenium adapter, {Page#body} gets refreshed on every call.
-    # Otherwise, subsequent DOM updates (i.e. JavaScript-induced) would be
-    # invisible.
-    # @return Page
-    def page
-      return @page unless self.class.config.http_adapter == :selenium
-
-      Page.new(
-        uri: @page.uri,
-        status_code: @page.uri,
-        body: driver.page_source,
-        headers: @page.headers
-      )
-    end
-
-    # The parsed response body.
-    # When using the Selenium adapter, this parses the body again on every call.
-    # Otherwise, subsequent DOM updates (i.e. JavaScript-induced) would be
-    # invisible.
-    # @method doc
-    # @see Page#doc
-    delegate doc: :page
-
-    # The Selenium WebDriver.
-    # @method driver
-    # @see https://github.com/peterc/pismo Pismo
-    # @see Page#driver
-    delegate driver: :adapter
-
-    # A Capybara driver that wraps the {#driver}.
-    # @method browser
-    # @see HTTPAdapters::SeleniumAdapter#browser
-    delegate browser: :adapter
-
-    # @method logger
-    delegate logger: :"self.class"
-  end
-end

data/lib/wayfarer/locals.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-require "concurrent"
-
-module Wayfarer
-  # @api private
-  module Locals
-    def self.thread_safe_counterpart(value)
-      case value
-      when Array      then Concurrent::Array.new(value)
-      when Hash       then Concurrent::Hash[value]
-      when TrueClass  then Concurrent::AtomicBoolean.new(value)
-      when FalseClass then Concurrent::AtomicBoolean.new(value)
-      when Integer then Concurrent::AtomicFixnum.new(value)
-      else value
-      end
-    end
-
-    def self.included(base)
-      base.extend(ClassMethods)
-    end
-
-    module ClassMethods
-      attr_reader :locals
-
-      def let(key)
-        raise "#let called without a block" unless block_given?
-        locals[key] = yield
-      end
-
-      def locals
-        @locals ||= {}
-      end
-
-      def locals=(locals)
-        @locals = locals
-      end
-    end
-  end
-end

data/lib/wayfarer/parsers/json_parser.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-require "oj" unless RUBY_PLATFORM == "java"
-
-module Wayfarer
-  module Parsers
-    # A wrapper class for parsing JSON.
-    # @private
-    module JSONParser
-      module_function
-
-      # Parses a JSON string.
-      # @param [String] json_str the JSON string to parse.
-      # @return [OpenStruct]
-      def parse(json_str)
-        RUBY_PLATFORM == "java" ? JSON.parse(json_str) : Oj.load(json_str)
-      end
-    end
-  end
-end

data/lib/wayfarer/parsers/xml_parser.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require "nokogiri"
-
-module Wayfarer
-  module Parsers
-    # A wrapper class for parsing HTML/XML.
-    # @private
-    module XMLParser
-      module_function
-
-      # Parses an XML string.
-      # @param [String] xml_str the XML string to parse.
-      # @return [Nokogiri::XML::Document]
-      def parse_xml(xml_str)
-        Nokogiri::XML(xml_str)
-      end
-
-      # Parses a HTML string.
-      # @param [String] html_str the HTML string to parse.
-      # @return [Nokogiri::HTML::Document]
-      def parse_html(html_str)
-        Nokogiri::HTML(html_str)
-      end
-    end
-  end
-end

data/lib/wayfarer/processor.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-require "pp"
-require "concurrent"
-require "observer"
-
-module Wayfarer
-  # Runs jobs.
-  class Processor
-    extend Forwardable
-
-    include Observable
-    include CrawlObserver::Events
-    include CrawlObserver::ObservableShortcuts
-
-    attr_reader :job
-
-    delegate config: :job
-    delegate logger: :config
-
-    def initialize(job, frontier, dispatcher)
-      @job = job
-      @frontier = frontier
-      @dispatcher = dispatcher
-      @halted = Concurrent::AtomicBoolean.new(false)
-    end
-
-    # Whether processing is done.
-    # @return [true, false]
-    def halted?
-      @halted.value
-    end
-
-    # Sets a halt flag.
-    def halt!
-      @halted.make_true
-    end
-
-    # Runs the job.
-    # @param [*Array<URI>, *Array<String>] uris
-    def run(*_uris)
-      notify_observers!(FirstCycle.new(@frontier))
-
-      while @halted.false? && @frontier.cycle
-        current_uris = @frontier.current_uris
-        queue = current_uris.inject(Queue.new, :push)
-
-        notify_observers!(NewCycle.new(current_uris.count))
-
-        @threads = Array.new(config.connection_count) do
-          Thread.new do
-            begin
-              loop do
-                uri = queue.pop(true)
-                break if uri.nil? || @halted.true?
-                handle_dispatch_result(@dispatcher.dispatch(@job, uri))
-              end
-            rescue ThreadError
-              notify_observers!(CycleFinished.new)
-            end
-          end
-        end
-
-        @threads.each(&:join)
-
-        notify_observers!(AboutToCycle.new(@frontier.staged_uris.count))
-      end
-    ensure
-      halt!
-      @frontier.free
-      @dispatcher.adapter_pool.free
-    end
-
-    private
-
-    def handle_dispatch_result(result)
-      case result
-      when Dispatcher::Mismatch then handle_mismatch(result)
-      when Dispatcher::Halt     then handle_halt(result)
-      when Dispatcher::Stage    then handle_stage(result)
-      when Dispatcher::Error    then handle_error(result)
-      end
-    end
-
-    def handle_mismatch(mismatch)
-      notify_observers!(MismatchedURI.new(mismatch.uri))
-    end
-
-    def handle_halt(halt)
-      notify_observers!(HaltInitiated.new(halt.action, halt.uri))
-      halt!
-    end
-
-    def handle_stage(stage)
-      notify_observers!(StagingURIs.new(stage.uris.count))
-      @frontier.stage(*stage.uris) unless halted?
-    end
-
-    def handle_error(error)
-      notify_observers!(UnhandledError.new(error.exception))
-    end
-  end
-end

data/lib/wayfarer/routing/custom_rule.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-require "uri"
-
-module Wayfarer
-  module Routing
-    # @private
-    class CustomRule < Rule
-      def initialize(delegate_or_block = proc, opts = {}, &proc)
-        @delegate_or_block = delegate_or_block
-        super(opts, &proc)
-      end
-
-      private
-
-      def match!(uri)
-        !!@delegate_or_block.call(uri)
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/filetypes_rule.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-require "uri"
-
-module Wayfarer
-  module Routing
-    # @private
-    class FiletypesRule < Rule
-      def initialize(types, opts = {}, &proc)
-        @types = types
-        super(opts, &proc)
-      end
-
-      private
-
-      def match!(uri)
-        @types.any? { |type| uri.path =~ /\.#{type}$/ }
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/host_rule.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Routing
-    # @private
-    class HostRule < Rule
-      def initialize(str_or_regexp, opts = {}, &proc)
-        @str_or_regexp = str_or_regexp
-        super(opts, &proc)
-      end
-
-      # rubocop:disable Style/CaseEquality
-      def match!(uri)
-        @str_or_regexp === uri.host
-      end
-      # rubocop:enable Style/CaseEquality
-    end
-  end
-end