wayfarer 0.0.3

data/lib/wayfarer/dispatcher.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/indifferent_access"
+
+module Wayfarer
+  # Creates job instances, retrieves pages and, if a URI matches a route, calls
+  # methods on the instances.
+  class Dispatcher
+    extend Forwardable
+
+    include Observable
+    include CrawlObserver::Events
+    include CrawlObserver::ObservableShortcuts
+
+    # Result types that a {Processor} operates with.
+    Mismatch = Struct.new(:uri)
+    Halt     = Struct.new(:uri, :action)
+    Stage    = Struct.new(:uris, :ret_val)
+    Error    = Struct.new(:exception)
+
+    # @!attribute [r] adapter_pool
+    # @return [AdapterPool]
+    attr_reader :adapter_pool
+
+    # @!attribute [r] job
+    attr_reader :job
+
+    def initialize(job)
+      @job = job
+      @adapter_pool = HTTPAdapters::AdapterPool.new(job)
+    end
+
+    delegate config: :job
+    delegate logger: :config
+
+    # Dispatches this URI. Matches an URI against the rules of the job's router.
+    # If a rule matches, the page is retrieved, and the action associated with
+    # the route is called.
+    #
+    # @param [Job] job
+    # @param [URI] uri
+    def dispatch(job, uri, is_peeking: false)
+      action, params = job.router.route(uri)
+      return Mismatch.new(uri) unless action
+
+      params = ActiveSupport::HashWithIndifferentAccess.new(params)
+
+      notify_observers!(DispatchedURI.new(action, uri))
+
+      job_instance = job.new
+      result = nil
+
+      adapter_pool.with do |adapter|
+        job_instance.page = adapter.fetch(uri)
+        job_instance.adapter = adapter
+        job_instance.params = params
+
+        result = job_instance.public_send(action) { |peek_uri|
+          begin
+            unless is_peeking
+              notify_observers!(Peeking.new(uri))
+              result = dispatch(job, URI(peek_uri), is_peeking: true)
+              result.ret_val
+            end
+          rescue
+            nil
+          end
+        }
+      end
+
+      if job_instance.halts?
+        Halt.new(uri, action)
+      else
+        Stage.new(job_instance.staged_uris, result)
+      end
+    # What follows are exceptions whose origin I don't care about at the moment
+    # TODO: Better logging
+    rescue Net::HTTP::Persistent::Error
+      logger.warn("Net::HTTP::Persistent::Error @ #{uri}")
+    rescue Errno::EHOSTUNREACH
+      logger.warn("Host unreachable @ #{uri}")
+    rescue Errno::ENETUNREACH
+      logger.warn("No route to network present @ #{uri}")
+    rescue Net::OpenTimeout, Net::ReadTimeout
+      logger.warn("::Net timeout @ #{uri}")
+
+    # SSL verification failed due to a missing certificate
+    rescue OpenSSL::SSL::SSLError
+      logger.warn("SSL verification failed @ #{uri}")
+
+    # Ruby/zlib encountered a Z_DATA_ERROR.
+    # Usually if a stream was prematurely freed.
+    # Probably has to do with net-http-persistent?
+    rescue Zlib::DataError
+      logger.warn("Z_DATA_ERROR")
+    rescue HTTPAdapters::NetHTTPAdapter::MalformedURI, URI::InvalidURIError
+      logger.info("[warn#{self}] Malformed URI @ #{uri}")
+    rescue HTTPAdapters::NetHTTPAdapter::MalformedRedirectURI
+      logger.info("Malformed redirect URI @ #{uri}")
+    rescue HTTPAdapters::NetHTTPAdapter::MaximumRedirectCountReached
+      logger.info("Maximum redirect count reached @ #{uri}")
+    end
+  end
+end

data/lib/wayfarer/finders.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Finders
+    # Returns the expanded `href` attribute URIs from all or targeted `<a>` tags.
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def links(*filters)
+      query("a", "href", *filters)
+    end
+
+    # Returns the expanded `href` attribute URIs from all or targeted `<link rel="stylesheet" ...>` tags.
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def stylesheets(*filters)
+      query("link[rel='stylesheet']", "href", *filters)
+    end
+
+    # Returns the expanded `src` attribute URIs from all or targeted `<script>` tags.
+    # TODO: Tests
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def javascripts(*filters)
+      query("script", "src", *filters)
+    end
+
+    alias scripts javascripts
+
+    # Returns the expanded `src` attribute URIs from all or targeted `<img>` tags.
+    # TODO: Tests
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def images(*filters)
+      query("img", "src", *filters)
+    end
+
+    private
+
+    # TODO: Lord have mercy
+    def query(selector, attr, *filters)
+      nodes = if filters.any?
+                doc.search(*filters).css(selector)
+              else
+                doc.css(selector)
+              end
+
+      links = nodes.map { |node|
+        begin
+          URI.join(uri, node.attr(attr))
+        rescue
+          nil
+        end
+      }
+
+      links
+        .find_all { |uri| uri.is_a?(URI) }
+        .uniq
+        .map(&:to_s)
+    end
+  end
+end

data/lib/wayfarer/frontiers/frontier.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Frontiers
+    # @abstract The common behaviour of all frontiers.
+    # @api private
+    class Frontier
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      # Returns URIs to be scraped in the current cycle.
+      # @note Usually an expensive operation!
+      # @return [Array<URI>]
+      def current_uris
+        raise "Unimplemented"
+      end
+
+      # Returns staged URIs.
+      # @return [Array<URI>]
+      def staged_uris
+        raise "Unimplemented"
+      end
+
+      # Stages URIs for processing in the next cycle.
+      # @param [*Array<URI>, *Array<String>] uris
+      def stage(*_uris)
+        raise "Unimplemented"
+      end
+
+      # Whether a URI is cached.
+      def staged?(_uri)
+        raise "Unimplemented"
+      end
+
+      # Caches URIs so they don't get processed again.
+      # @param [*Array<URI>, *Array<String>] uris
+      def cache(*_uris)
+        raise "Unimplemented"
+      end
+
+      # Whether a URI is cached.
+      def cached?(_uri)
+        raise "Unimplemented"
+      end
+
+      # Frees resources.
+      def free; end
+
+      # TODO: Documentation
+      def cycle
+        unless config.allow_circulation
+          cache(*current_uris) # TODO: Make it a template method
+          filter_staged_uris!
+        end
+
+        return false if staged_uris.none?
+
+        swap!
+        reset_staged_uris!
+
+        true
+      end
+
+      protected
+
+      # TODO: Documentation
+      def filter_staged_uris!; end
+
+      # TODO: Documentation
+      def swap!; end
+
+      # TODO: Documentation
+      def reset_staged_uris!; end
+    end
+  end
+end

data/lib/wayfarer/frontiers/memory_bloomfilter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "bloomfilter-rb"
+
+module Wayfarer
+  module Frontiers
+    # An in-memory bloomfilter.
+    # @api private
+    class MemoryBloomfilter < MemoryFrontier
+      def initialize(config)
+        @filter = BloomFilter::Native.new(config.bloomfilter_opts)
+        super(config)
+      end
+
+      # @override
+      def cache(*uris)
+        uris.each { |uri| @filter.insert(uri) }
+      end
+
+      # @override
+      def cached?(uri)
+        @filter.include?(uri)
+      end
+
+      # Frees up memory.
+      def free
+        @filter.clear
+        super
+      end
+    end
+  end
+end

data/lib/wayfarer/frontiers/memory_frontier.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "set"
+require "parallel" unless JAVA_PLATFORM = "java"
+
+module Wayfarer
+  module Frontiers
+    # A naive in-memory frontier.
+    # @api private
+    class MemoryFrontier < Frontier
+      def initialize(config)
+        @current_uris = Set.new([])
+        @staged_uris  = Set.new([])
+        @cached_uris  = Set.new([])
+        super(config)
+      end
+
+      # @override
+      def current_uris
+        if JAVA_PLATFORM == "java"
+          @current_uris.map { |uri| URI(uri) }
+        else
+          Parallel.map(@current_uris) { |uri| URI(uri) }
+        end
+      end
+
+      # @override
+      def staged_uris
+        @staged_uris.to_a # These are assumed to be URIs already, so no map
+      end
+
+      # @override
+      def stage(*uris)
+        @staged_uris |= uris
+      end
+
+      # @override
+      def staged?(uri)
+        @staged_uris.include?(uri.to_s)
+      end
+
+      # @override
+      def cache(*uris)
+        @cached_uris |= if JAVA_PLATFORM == "java"
+                          uris.map(&:to_s)
+                        else
+                          Parallel.map(uris, &:to_s)
+                        end
+      end
+
+      # @override
+      def cached?(uri)
+        @cached_uris.include?(uri.to_s)
+      end
+
+      # @override
+      def free
+        @current_uris = @staged_uris = @cached_uris = nil
+      end
+
+      private
+
+      def reset_staged_uris!
+        @staged_uris = Set.new([])
+      end
+
+      def swap!
+        @current_uris = @staged_uris
+      end
+
+      def filter_staged_uris!
+        @staged_uris.delete_if { |uri| cached?(uri) }
+      end
+    end
+  end
+end

data/lib/wayfarer/frontiers/memory_trie_frontier.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "trie"
+
+module Wayfarer
+  module Frontiers
+    # An in-memory trie.
+    # @api private
+    class MemoryTrieFrontier < MemoryFrontier
+      def initialize(config)
+        @trie = Trie.new
+        super(config)
+      end
+
+      # @override
+      def cache(*uris)
+        uris.each { |uri| @trie.add(uri.to_s) }
+      end
+
+      # @override
+      def match!(uri)
+        @str_or_regexp === uri.host
+      end
+
+      def cached?(uri)
+        # RuboCop autocorrects `#has_key?` to `#key?` otherwise
+        # rubocop:disable Style/PreferredHashMethods
+        @trie.has_key?(uri.to_s)
+        # rubocop:enable Style/PreferredHashMethods
+      end
+
+      # @override
+      def free
+        @trie = nil
+        super
+      end
+    end
+  end
+end

data/lib/wayfarer/frontiers/normalize_uris.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "normalize_url"
+
+module Wayfarer
+  module Frontiers
+    # @api private
+    module NormalizeURIs
+      # @override
+      def stage(*uris)
+        super(*uris.map { |uri| normalize(uri) })
+      end
+
+      # @override
+      def staged?(uri)
+        super(normalize(uri))
+      end
+
+      # @override
+      def cache(*uris)
+        super(*uris.map { |uri| normalize(uri) })
+      end
+
+      # @override
+      def cached?(uri)
+        super(normalize(uri))
+      end
+
+      private
+
+      def normalize(uri)
+        NormalizeUrl.process(uri, config.normalize_uri_options)
+      end
+
+      def to_s
+        "URI-normalizing #{super}"
+      end
+
+      def method_missing(*argv, &proc)
+        super(*argv, &proc)
+      end
+
+      def respond_to_missing?(method, private = false)
+        @frontier.respond_to?(method) || super
+      end
+    end
+  end
+end

data/lib/wayfarer/frontiers/redis_bloomfilter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "bloomfilter-rb"
+
+module Wayfarer
+  module Frontiers
+    # A Redis bloomfilter.
+    # @api private
+    class RedisBloomfilter < MemoryBloomfilter
+      def initialize(config)
+        @conn = Redis.new(config.redis_opts)
+        @filter = BloomFilter::Redis.new(config.bloomfilter_opts.merge(db: @conn))
+        super(config)
+      end
+    end
+
+    # @override
+    def cache(*uris)
+      uris.each { |uri| @filter.insert(uri) }
+    end
+
+    # @override
+    def cached?(uri)
+      @filter.include?(uri)
+    end
+
+    # @override
+    def free
+      @filter.clear
+      @conn.disconnect!
+      super
+    end
+  end
+end

data/lib/wayfarer/frontiers/redis_frontier.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "redis"
+
+module Wayfarer
+  module Frontiers
+    # A Redis frontier
+    # @api private
+    class RedisFrontier < Frontier
+      def initialize(config)
+        @conn = Redis.new(config.redis_opts)
+        super(config)
+      end
+
+      # @override
+      def current_uris
+        @conn.smembers(current_uris_key).map { |str| URI(str) }
+      end
+
+      # @override
+      def staged_uris
+        @conn.smembers(staged_uris_key).map { |str| URI(str) }
+      end
+
+      # @override
+      def stage(*uris)
+        @conn.sadd(staged_uris_key, uris.map(&:to_s)) if uris.any?
+      end
+
+      # @override
+      def staged?(uri)
+        @conn.sismember(staged_uris_key, uri.to_s)
+      end
+
+      # @override
+      def cache(*uris)
+        @conn.sadd(cached_uris_key, uris.map(&:to_s)) if uris.any?
+      end
+
+      # @override
+      def cached?(uri)
+        @conn.sismember(cached_uris_key, uri.to_s)
+      end
+
+      # @override
+      def free
+        [current_uris_key, staged_uris_key, cached_uris_key].each do |key|
+          @conn.del(key)
+        end
+
+        @conn.disconnect!
+      end
+
+      private
+
+      def reset_staged_uris!
+        @conn.del(staged_uris_key)
+      end
+
+      # @override
+      def swap!
+        # Achieve: @current_uris = @staged_uris
+        @conn.rename(staged_uris_key, current_uris_key)
+      end
+
+      def filter_staged_uris!
+        @conn.sdiffstore(staged_uris_key, staged_uris_key, cached_uris_key)
+      end
+
+      def current_uris_key
+        "#{@config.uuid}_current_uris"
+      end
+
+      def staged_uris_key
+        "#{@config.uuid}_staged_uris"
+      end
+
+      def cached_uris_key
+        "#{@config.uuid}_cached_uris"
+      end
+    end
+  end
+end

data/lib/wayfarer/http_adapters/adapter_pool.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "connection_pool"
+
+module Wayfarer
+  module HTTPAdapters
+    # A connection pool that hands out HTTP adapters.
+    # @private
+    class AdapterPool
+      extend Forwardable
+
+      def initialize(job)
+        @job = job
+        @config = job.config
+
+        size = @config.connection_count
+        timeout = @config.connection_timeout
+
+        @pool = ConnectionPool.new(
+          size: size,
+          timeout: timeout,
+          &method(:instantiate_adapter)
+        )
+      end
+
+      # Shuts down all HTTP adapters
+      def free
+        @pool.shutdown(&:free)
+      end
+
+      private
+
+      def instantiate_adapter
+        adapter = if @config.http_adapter == :selenium
+                    HTTPAdapters::SeleniumAdapter.new(@config)
+                  else
+                    HTTPAdapters::NetHTTPAdapter.instance(@config)
+                  end
+
+        @job.run_hook(
+          :setup_adapter,
+          adapter,
+          adapter.try(:driver),
+          adapter.try(:browser)
+        )
+
+        adapter
+      end
+
+      def method_missing(method, *argv, &proc)
+        super if method == :shutdown
+        @pool.public_send(method, *argv, &proc)
+      end
+
+      def respond_to_missing?(method, private = false)
+        return false if method == :shutdown
+        @pool.respond_to?(method) || super
+      end
+    end
+  end
+end

data/lib/wayfarer/http_adapters/net_http_adapter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "net/http"
+require "net/http/persistent"
+
+module Wayfarer
+  module HTTPAdapters
+    # A singleton adapter for net-http-persistent.
+    # @api private
+    class NetHTTPAdapter
+      # Supported standard lib classes
+      RECOGNIZED_URI_TYPES = [
+        URI::HTTP,
+        URI::HTTPS
+      ].freeze
+
+      MalformedURI = Class.new(StandardError)
+      MalformedRedirectURI = Class.new(StandardError)
+      MaximumRedirectCountReached = Class.new(StandardError)
+
+      attr_accessor :request_header_overrides
+
+      # TODO: Remove default parameter value
+      def self.instance(config = Wayfarer.config)
+        @@instance ||= new(config)
+      end
+
+      def initialize(config)
+        @config = config
+        @conn = Net::HTTP::Persistent.new("wayfarer-#{SecureRandom.uuid}")
+      end
+
+      # This is a singleton class. Use ::instance instead.
+      private_class_method :new
+
+      # Fetches a page.
+      # @return [Page]
+      # @raise [MalformedURI] if the URI is not supported.
+      # @raise [MalformedRedirectURI] if a redirection URI is not supported.
+      # @raise [MaximumRedirectCountReached] if too many redirections are
+      # encountered.
+      def fetch(uri, redirects_followed = 0)
+        if !RECOGNIZED_URI_TYPES.include?(uri.class)
+          raise _ = if redirects_followed.positive?
+                      MalformedRedirectURI
+                    else
+                      MalformedURI
+                    end
+        elsif redirects_followed > @config.max_http_redirects
+          raise MaximumRedirectCountReached
+        end
+
+        res = @conn.request(uri)
+
+        if res.is_a? Net::HTTPRedirection
+          redirect_uri = URI(res["location"])
+          return fetch(redirect_uri, redirects_followed + 1)
+        end
+
+        Page.new(
+          uri: uri,
+          status_code: res.code.to_i,
+          body: res.body,
+          headers: res.to_hash
+        )
+      rescue SocketError
+        raise MalformedURI
+      end
+
+      # Shuts down all connections.
+      def free
+        @conn.shutdown
+      end
+    end
+  end
+end