wayfarer-jruby 0.0.1

data/lib/wayfarer/dispatcher.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/indifferent_access"
+
+module Wayfarer
+  # Creates job instances, retrieves pages and, if a URI matches a route, calls
+  # methods on the instances.
+  class Dispatcher
+    extend Forwardable
+
+    include Observable
+    include CrawlObserver::Events
+    include CrawlObserver::ObservableShortcuts
+
+    # Result types that a {Processor} operates with.
+    Mismatch = Struct.new(:uri)
+    Halt     = Struct.new(:uri, :action)
+    Stage    = Struct.new(:uris, :ret_val)
+    Error    = Struct.new(:exception)
+
+    # @!attribute [r] adapter_pool
+    # @return [AdapterPool]
+    attr_reader :adapter_pool
+
+    # @!attribute [r] job
+    attr_reader :job
+
+    def initialize(job)
+      @job = job
+      @adapter_pool = HTTPAdapters::AdapterPool.new(job)
+    end
+
+    delegate config: :job
+    delegate logger: :config
+
+    # Dispatches this URI. Matches an URI against the rules of the job's router.
+    # If a rule matches, the page is retrieved, and the action associated with
+    # the route is called.
+    #
+    # @param [Job] job
+    # @param [URI] uri
+    def dispatch(job, uri, is_peeking: false)
+      action, params = job.router.route(uri)
+      return Mismatch.new(uri) unless action
+
+      params = ActiveSupport::HashWithIndifferentAccess.new(params)
+
+      notify_observers!(DispatchedURI.new(action, uri))
+
+      job_instance = job.new
+      result = nil
+
+      adapter_pool.with do |adapter|
+        job_instance.page = adapter.fetch(uri)
+        job_instance.adapter = adapter
+        job_instance.params = params
+
+        result = job_instance.public_send(action) { |peek_uri|
+          begin
+            unless is_peeking
+              notify_observers!(Peeking.new(uri))
+              result = dispatch(job, URI(peek_uri), is_peeking: true)
+              result.ret_val
+            end
+          rescue
+            nil
+          end
+        }
+      end
+
+      if job_instance.halts?
+        Halt.new(uri, action)
+      else
+        Stage.new(job_instance.staged_uris, result)
+      end
+    # What follows are exceptions whose origin I don't care about at the moment
+    # TODO: Better logging
+    rescue Net::HTTP::Persistent::Error
+      logger.warn("Net::HTTP::Persistent::Error @ #{uri}")
+    rescue Errno::EHOSTUNREACH
+      logger.warn("Host unreachable @ #{uri}")
+    rescue Errno::ENETUNREACH
+      logger.warn("No route to network present @ #{uri}")
+    rescue Net::OpenTimeout, Net::ReadTimeout
+      logger.warn("::Net timeout @ #{uri}")
+
+    # SSL verification failed due to a missing certificate
+    rescue OpenSSL::SSL::SSLError
+      logger.warn("SSL verification failed @ #{uri}")
+
+    # Ruby/zlib encountered a Z_DATA_ERROR.
+    # Usually if a stream was prematurely freed.
+    # Probably has to do with net-http-persistent?
+    rescue Zlib::DataError
+      logger.warn("Z_DATA_ERROR")
+    rescue HTTPAdapters::NetHTTPAdapter::MalformedURI, URI::InvalidURIError
+      logger.info("[warn#{self}] Malformed URI @ #{uri}")
+    rescue HTTPAdapters::NetHTTPAdapter::MalformedRedirectURI
+      logger.info("Malformed redirect URI @ #{uri}")
+    rescue HTTPAdapters::NetHTTPAdapter::MaximumRedirectCountReached
+      logger.info("Maximum redirect count reached @ #{uri}")
+    end
+  end
+end

data/lib/wayfarer/finders.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Finders
+    # Returns the expanded `href` attribute URIs from all or targeted `<a>` tags.
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def links(*filters)
+      query("a", "href", *filters)
+    end
+
+    # Returns the expanded `href` attribute URIs from all or targeted `<link rel="stylesheet" ...>` tags.
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def stylesheets(*filters)
+      query("link[rel='stylesheet']", "href", *filters)
+    end
+
+    # Returns the expanded `src` attribute URIs from all or targeted `<script>` tags.
+    # TODO: Tests
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def javascripts(*filters)
+      query("script", "src", *filters)
+    end
+
+    alias scripts javascripts
+
+    # Returns the expanded `src` attribute URIs from all or targeted `<img>` tags.
+    # TODO: Tests
+    # @param [*Array<String>] filters CSS/XPath expressions.
+    # @return [Array<URI>]
+    def images(*filters)
+      query("img", "src", *filters)
+    end
+
+    private
+
+    # TODO: Lord have mercy
+    def query(selector, attr, *filters)
+      nodes = if filters.any?
+                doc.search(*filters).css(selector)
+              else
+                doc.css(selector)
+              end
+
+      links = nodes.map { |node|
+        begin
+          URI.join(uri, node.attr(attr))
+        rescue
+          nil
+        end
+      }
+
+      links
+        .find_all { |uri| uri.is_a?(URI) }
+        .uniq
+        .map(&:to_s)
+    end
+  end
+end

data/lib/wayfarer/frontiers/frontier.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Frontiers
+    # @abstract The common behaviour of all frontiers.
+    # @api private
+    class Frontier
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      # Returns URIs to be scraped in the current cycle.
+      # @note Usually an expensive operation!
+      # @return [Array<URI>]
+      def current_uris
+        raise "Unimplemented"
+      end
+
+      # Returns staged URIs.
+      # @return [Array<URI>]
+      def staged_uris
+        raise "Unimplemented"
+      end
+
+      # Stages URIs for processing in the next cycle.
+      # @param [*Array<URI>, *Array<String>] uris
+      def stage(*_uris)
+        raise "Unimplemented"
+      end
+
+      # Whether a URI is cached.
+      def staged?(_uri)
+        raise "Unimplemented"
+      end
+
+      # Caches URIs so they don't get processed again.
+      # @param [*Array<URI>, *Array<String>] uris
+      def cache(*_uris)
+        raise "Unimplemented"
+      end
+
+      # Whether a URI is cached.
+      def cached?(_uri)
+        raise "Unimplemented"
+      end
+
+      # Frees resources.
+      def free; end
+
+      # TODO: Documentation
+      def cycle
+        unless config.allow_circulation
+          cache(*current_uris) # TODO: Make it a template method
+          filter_staged_uris!
+        end
+
+        return false if staged_uris.none?
+
+        swap!
+        reset_staged_uris!
+
+        true
+      end
+
+      protected
+
+      # TODO: Documentation
+      def filter_staged_uris!; end
+
+      # TODO: Documentation
+      def swap!; end
+
+      # TODO: Documentation
+      def reset_staged_uris!; end
+    end
+  end
+end

data/lib/wayfarer/frontiers/memory_bloomfilter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "bloomfilter-rb"
+
+module Wayfarer
+  module Frontiers
+    # An in-memory bloomfilter.
+    # @api private
+    class MemoryBloomfilter < MemoryFrontier
+      def initialize(config)
+        @filter = BloomFilter::Native.new(config.bloomfilter_opts)
+        super(config)
+      end
+
+      # @override
+      def cache(*uris)
+        uris.each { |uri| @filter.insert(uri) }
+      end
+
+      # @override
+      def cached?(uri)
+        @filter.include?(uri)
+      end
+
+      # Frees up memory.
+      def free
+        @filter.clear
+        super
+      end
+    end
+  end
+end

data/lib/wayfarer/frontiers/memory_frontier.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "set"
+require "parallel" unless JAVA_PLATFORM = "java"
+
+module Wayfarer
+  module Frontiers
+    # A naive in-memory frontier.
+    # @api private
+    class MemoryFrontier < Frontier
+      def initialize(config)
+        @current_uris = Set.new([])
+        @staged_uris  = Set.new([])
+        @cached_uris  = Set.new([])
+        super(config)
+      end
+
+      # @override
+      def current_uris
+        if JAVA_PLATFORM == "java"
+          @current_uris.map { |uri| URI(uri) }
+        else
+          Parallel.map(@current_uris) { |uri| URI(uri) }
+        end
+      end
+
+      # @override
+      def staged_uris
+        @staged_uris.to_a # These are assumed to be URIs already, so no map
+      end
+
+      # @override
+      def stage(*uris)
+        @staged_uris |= uris
+      end
+
+      # @override
+      def staged?(uri)
+        @staged_uris.include?(uri.to_s)
+      end
+
+      # @override
+      def cache(*uris)
+        @cached_uris |= if JAVA_PLATFORM == "java"
+                          uris.map(&:to_s)
+                        else
+                          Parallel.map(uris, &:to_s)
+                        end
+      end
+
+      # @override
+      def cached?(uri)
+        @cached_uris.include?(uri.to_s)
+      end
+
+      # @override
+      def free
+        @current_uris = @staged_uris = @cached_uris = nil
+      end
+
+      private
+
+      def reset_staged_uris!
+        @staged_uris = Set.new([])
+      end
+
+      def swap!
+        @current_uris = @staged_uris
+      end
+
+      def filter_staged_uris!
+        @staged_uris.delete_if { |uri| cached?(uri) }
+      end
+    end
+  end
+end

data/lib/wayfarer/frontiers/memory_trie_frontier.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "trie"
+
+module Wayfarer
+  module Frontiers
+    # An in-memory trie.
+    # @api private
+    class MemoryTrieFrontier < MemoryFrontier
+      def initialize(config)
+        @trie = Trie.new
+        super(config)
+      end
+
+      # @override
+      def cache(*uris)
+        uris.each { |uri| @trie.add(uri.to_s) }
+      end
+
+      # @override
+      def match!(uri)
+        @str_or_regexp === uri.host
+      end
+
+      def cached?(uri)
+        # RuboCop autocorrects `#has_key?` to `#key?` otherwise
+        # rubocop:disable Style/PreferredHashMethods
+        @trie.has_key?(uri.to_s)
+        # rubocop:enable Style/PreferredHashMethods
+      end
+
+      # @override
+      def free
+        @trie = nil
+        super
+      end
+    end
+  end
+end

data/lib/wayfarer/frontiers/normalize_uris.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "normalize_url"
+
+module Wayfarer
+  module Frontiers
+    # @api private
+    module NormalizeURIs
+      # @override
+      def stage(*uris)
+        super(*uris.map { |uri| normalize(uri) })
+      end
+
+      # @override
+      def staged?(uri)
+        super(normalize(uri))
+      end
+
+      # @override
+      def cache(*uris)
+        super(*uris.map { |uri| normalize(uri) })
+      end
+
+      # @override
+      def cached?(uri)
+        super(normalize(uri))
+      end
+
+      private
+
+      def normalize(uri)
+        NormalizeUrl.process(uri, config.normalize_uri_options)
+      end
+
+      def to_s
+        "URI-normalizing #{super}"
+      end
+
+      def method_missing(*argv, &proc)
+        super(*argv, &proc)
+      end
+
+      def respond_to_missing?(method, private = false)
+        @frontier.respond_to?(method) || super
+      end
+    end
+  end
+end