wayfarer 0.0.3 → 0.4.0

data/lib/wayfarer/crawl_observer.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  class CrawlObserver
-    module Events
-      FirstCycle = Struct.new(:frontier)
-      NewCycle = Struct.new(:current_uris_count)
-      DispatchedURI = Struct.new(:action, :uri)
-      CycleFinished = Class.new
-      Peeking = Struct.new(:uri)
-      AboutToCycle = Struct.new(:staged_uris_count)
-      MismatchedURI = Struct.new(:uri)
-      HaltInitiated = Struct.new(:action, :uri)
-      StagingURIs = Struct.new(:staged_uris_count)
-      UnhandledError = Struct.new(:exception)
-    end
-
-    module ObservableShortcuts
-      def notify_observers!(*argv)
-        changed
-        notify_observers(*argv)
-      end
-    end
-
-    extend Forwardable
-
-    attr_reader :logger
-
-    def initialize(*observables, logger)
-      @logger = logger
-      observables.each { |obsv| obsv.add_observer(self) }
-    end
-
-    def update(event)
-      case event
-      when Events::FirstCycle     then first_cycle(event)
-      when Events::NewCycle       then new_cycle(event)
-      when Events::DispatchedURI  then dispatched_uri(event)
-      when Events::CycleFinished  then cycle_finished
-      when Events::Peeking        then peeking(event)
-      when Events::AboutToCycle   then about_to_cycle(event)
-      when Events::MismatchedURI  then mismatched_uri(event)
-      when Events::HaltInitiated  then halt_initiated(event)
-      when Events::StagingURIs    then staging_uris(event)
-      when Events::UnhandledError then unhandled_error(event)
-      end
-    end
-
-    private
-
-    def first_cycle(event)
-      logger.info("First cycle")
-      logger.info("Frontier: #{event.frontier}")
-    end
-
-    def new_cycle(event)
-      logger.info("Current cycle contains #{event.current_uris_count} URI(s)")
-    end
-
-    def dispatched_uri(event)
-      logger.info("Dispatched to \##{event.action}: #{event.uri}")
-    end
-
-    def cycle_finished
-      logger.info("No URIs left in current cycle")
-    end
-
-    def peeking(event)
-      logger.info("Peeking into: #{event.uri}")
-    end
-
-    def about_to_cycle(event)
-      logger.info("About to cycle. #{event.staged_uris_count} staged URI(s)")
-    end
-
-    def mismatched_uri(event)
-      logger.debug("No matching route for: #{event.uri}")
-    end
-
-    def halt_initiated(event)
-      logger.info("Halt initiated from \##{event.action} at: #{event.uri}")
-    end
-
-    def staging_uris(event)
-      logger.info("Staging #{event.staged_uris_count} URI(s)")
-    end
-
-    def unhandled_error(event)
-      level = config.reraise_exceptions ? :fatal : :error
-
-      if config.print_stacktraces
-        logger.public_send level, <<~LOGGER
-          Unhandled exception in an action: #{event.exception.class.inspect}
-          #{event.exception.backtrace.map(&:to_s).join("\n* ")}
-        LOGGER
-      else
-        logger.public_send level, <<~LOGGER
-          Unhandled exception in an action: #{event.exception.class.inspect}
-        LOGGER
-      end
-    end
-  end
-end

data/lib/wayfarer/dispatcher.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-require "active_support/core_ext/hash/indifferent_access"
-
-module Wayfarer
-  # Creates job instances, retrieves pages and, if a URI matches a route, calls
-  # methods on the instances.
-  class Dispatcher
-    extend Forwardable
-
-    include Observable
-    include CrawlObserver::Events
-    include CrawlObserver::ObservableShortcuts
-
-    # Result types that a {Processor} operates with.
-    Mismatch = Struct.new(:uri)
-    Halt     = Struct.new(:uri, :action)
-    Stage    = Struct.new(:uris, :ret_val)
-    Error    = Struct.new(:exception)
-
-    # @!attribute [r] adapter_pool
-    # @return [AdapterPool]
-    attr_reader :adapter_pool
-
-    # @!attribute [r] job
-    attr_reader :job
-
-    def initialize(job)
-      @job = job
-      @adapter_pool = HTTPAdapters::AdapterPool.new(job)
-    end
-
-    delegate config: :job
-    delegate logger: :config
-
-    # Dispatches this URI. Matches an URI against the rules of the job's router.
-    # If a rule matches, the page is retrieved, and the action associated with
-    # the route is called.
-    #
-    # @param [Job] job
-    # @param [URI] uri
-    def dispatch(job, uri, is_peeking: false)
-      action, params = job.router.route(uri)
-      return Mismatch.new(uri) unless action
-
-      params = ActiveSupport::HashWithIndifferentAccess.new(params)
-
-      notify_observers!(DispatchedURI.new(action, uri))
-
-      job_instance = job.new
-      result = nil
-
-      adapter_pool.with do |adapter|
-        job_instance.page = adapter.fetch(uri)
-        job_instance.adapter = adapter
-        job_instance.params = params
-
-        result = job_instance.public_send(action) { |peek_uri|
-          begin
-            unless is_peeking
-              notify_observers!(Peeking.new(uri))
-              result = dispatch(job, URI(peek_uri), is_peeking: true)
-              result.ret_val
-            end
-          rescue
-            nil
-          end
-        }
-      end
-
-      if job_instance.halts?
-        Halt.new(uri, action)
-      else
-        Stage.new(job_instance.staged_uris, result)
-      end
-    # What follows are exceptions whose origin I don't care about at the moment
-    # TODO: Better logging
-    rescue Net::HTTP::Persistent::Error
-      logger.warn("Net::HTTP::Persistent::Error @ #{uri}")
-    rescue Errno::EHOSTUNREACH
-      logger.warn("Host unreachable @ #{uri}")
-    rescue Errno::ENETUNREACH
-      logger.warn("No route to network present @ #{uri}")
-    rescue Net::OpenTimeout, Net::ReadTimeout
-      logger.warn("::Net timeout @ #{uri}")
-
-    # SSL verification failed due to a missing certificate
-    rescue OpenSSL::SSL::SSLError
-      logger.warn("SSL verification failed @ #{uri}")
-
-    # Ruby/zlib encountered a Z_DATA_ERROR.
-    # Usually if a stream was prematurely freed.
-    # Probably has to do with net-http-persistent?
-    rescue Zlib::DataError
-      logger.warn("Z_DATA_ERROR")
-    rescue HTTPAdapters::NetHTTPAdapter::MalformedURI, URI::InvalidURIError
-      logger.info("[warn#{self}] Malformed URI @ #{uri}")
-    rescue HTTPAdapters::NetHTTPAdapter::MalformedRedirectURI
-      logger.info("Malformed redirect URI @ #{uri}")
-    rescue HTTPAdapters::NetHTTPAdapter::MaximumRedirectCountReached
-      logger.info("Maximum redirect count reached @ #{uri}")
-    end
-  end
-end

data/lib/wayfarer/finders.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Finders
-    # Returns the expanded `href` attribute URIs from all or targeted `<a>` tags.
-    # @param [*Array<String>] filters CSS/XPath expressions.
-    # @return [Array<URI>]
-    def links(*filters)
-      query("a", "href", *filters)
-    end
-
-    # Returns the expanded `href` attribute URIs from all or targeted `<link rel="stylesheet" ...>` tags.
-    # @param [*Array<String>] filters CSS/XPath expressions.
-    # @return [Array<URI>]
-    def stylesheets(*filters)
-      query("link[rel='stylesheet']", "href", *filters)
-    end
-
-    # Returns the expanded `src` attribute URIs from all or targeted `<script>` tags.
-    # TODO: Tests
-    # @param [*Array<String>] filters CSS/XPath expressions.
-    # @return [Array<URI>]
-    def javascripts(*filters)
-      query("script", "src", *filters)
-    end
-
-    alias scripts javascripts
-
-    # Returns the expanded `src` attribute URIs from all or targeted `<img>` tags.
-    # TODO: Tests
-    # @param [*Array<String>] filters CSS/XPath expressions.
-    # @return [Array<URI>]
-    def images(*filters)
-      query("img", "src", *filters)
-    end
-
-    private
-
-    # TODO: Lord have mercy
-    def query(selector, attr, *filters)
-      nodes = if filters.any?
-                doc.search(*filters).css(selector)
-              else
-                doc.css(selector)
-              end
-
-      links = nodes.map { |node|
-        begin
-          URI.join(uri, node.attr(attr))
-        rescue
-          nil
-        end
-      }
-
-      links
-        .find_all { |uri| uri.is_a?(URI) }
-        .uniq
-        .map(&:to_s)
-    end
-  end
-end

data/lib/wayfarer/frontiers/frontier.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-module Wayfarer
-  module Frontiers
-    # @abstract The common behaviour of all frontiers.
-    # @api private
-    class Frontier
-      attr_reader :config
-
-      def initialize(config)
-        @config = config
-      end
-
-      # Returns URIs to be scraped in the current cycle.
-      # @note Usually an expensive operation!
-      # @return [Array<URI>]
-      def current_uris
-        raise "Unimplemented"
-      end
-
-      # Returns staged URIs.
-      # @return [Array<URI>]
-      def staged_uris
-        raise "Unimplemented"
-      end
-
-      # Stages URIs for processing in the next cycle.
-      # @param [*Array<URI>, *Array<String>] uris
-      def stage(*_uris)
-        raise "Unimplemented"
-      end
-
-      # Whether a URI is cached.
-      def staged?(_uri)
-        raise "Unimplemented"
-      end
-
-      # Caches URIs so they don't get processed again.
-      # @param [*Array<URI>, *Array<String>] uris
-      def cache(*_uris)
-        raise "Unimplemented"
-      end
-
-      # Whether a URI is cached.
-      def cached?(_uri)
-        raise "Unimplemented"
-      end
-
-      # Frees resources.
-      def free; end
-
-      # TODO: Documentation
-      def cycle
-        unless config.allow_circulation
-          cache(*current_uris) # TODO: Make it a template method
-          filter_staged_uris!
-        end
-
-        return false if staged_uris.none?
-
-        swap!
-        reset_staged_uris!
-
-        true
-      end
-
-      protected
-
-      # TODO: Documentation
-      def filter_staged_uris!; end
-
-      # TODO: Documentation
-      def swap!; end
-
-      # TODO: Documentation
-      def reset_staged_uris!; end
-    end
-  end
-end

data/lib/wayfarer/frontiers/memory_bloomfilter.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-require "bloomfilter-rb"
-
-module Wayfarer
-  module Frontiers
-    # An in-memory bloomfilter.
-    # @api private
-    class MemoryBloomfilter < MemoryFrontier
-      def initialize(config)
-        @filter = BloomFilter::Native.new(config.bloomfilter_opts)
-        super(config)
-      end
-
-      # @override
-      def cache(*uris)
-        uris.each { |uri| @filter.insert(uri) }
-      end
-
-      # @override
-      def cached?(uri)
-        @filter.include?(uri)
-      end
-
-      # Frees up memory.
-      def free
-        @filter.clear
-        super
-      end
-    end
-  end
-end

data/lib/wayfarer/frontiers/memory_frontier.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-require "set"
-require "parallel" unless JAVA_PLATFORM = "java"
-
-module Wayfarer
-  module Frontiers
-    # A naive in-memory frontier.
-    # @api private
-    class MemoryFrontier < Frontier
-      def initialize(config)
-        @current_uris = Set.new([])
-        @staged_uris  = Set.new([])
-        @cached_uris  = Set.new([])
-        super(config)
-      end
-
-      # @override
-      def current_uris
-        if JAVA_PLATFORM == "java"
-          @current_uris.map { |uri| URI(uri) }
-        else
-          Parallel.map(@current_uris) { |uri| URI(uri) }
-        end
-      end
-
-      # @override
-      def staged_uris
-        @staged_uris.to_a # These are assumed to be URIs already, so no map
-      end
-
-      # @override
-      def stage(*uris)
-        @staged_uris |= uris
-      end
-
-      # @override
-      def staged?(uri)
-        @staged_uris.include?(uri.to_s)
-      end
-
-      # @override
-      def cache(*uris)
-        @cached_uris |= if JAVA_PLATFORM == "java"
-                          uris.map(&:to_s)
-                        else
-                          Parallel.map(uris, &:to_s)
-                        end
-      end
-
-      # @override
-      def cached?(uri)
-        @cached_uris.include?(uri.to_s)
-      end
-
-      # @override
-      def free
-        @current_uris = @staged_uris = @cached_uris = nil
-      end
-
-      private
-
-      def reset_staged_uris!
-        @staged_uris = Set.new([])
-      end
-
-      def swap!
-        @current_uris = @staged_uris
-      end
-
-      def filter_staged_uris!
-        @staged_uris.delete_if { |uri| cached?(uri) }
-      end
-    end
-  end
-end

data/lib/wayfarer/frontiers/memory_trie_frontier.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require "trie"
-
-module Wayfarer
-  module Frontiers
-    # An in-memory trie.
-    # @api private
-    class MemoryTrieFrontier < MemoryFrontier
-      def initialize(config)
-        @trie = Trie.new
-        super(config)
-      end
-
-      # @override
-      def cache(*uris)
-        uris.each { |uri| @trie.add(uri.to_s) }
-      end
-
-      # @override
-      def match!(uri)
-        @str_or_regexp === uri.host
-      end
-
-      def cached?(uri)
-        # RuboCop autocorrects `#has_key?` to `#key?` otherwise
-        # rubocop:disable Style/PreferredHashMethods
-        @trie.has_key?(uri.to_s)
-        # rubocop:enable Style/PreferredHashMethods
-      end
-
-      # @override
-      def free
-        @trie = nil
-        super
-      end
-    end
-  end
-end

data/lib/wayfarer/frontiers/normalize_uris.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-require "normalize_url"
-
-module Wayfarer
-  module Frontiers
-    # @api private
-    module NormalizeURIs
-      # @override
-      def stage(*uris)
-        super(*uris.map { |uri| normalize(uri) })
-      end
-
-      # @override
-      def staged?(uri)
-        super(normalize(uri))
-      end
-
-      # @override
-      def cache(*uris)
-        super(*uris.map { |uri| normalize(uri) })
-      end
-
-      # @override
-      def cached?(uri)
-        super(normalize(uri))
-      end
-
-      private
-
-      def normalize(uri)
-        NormalizeUrl.process(uri, config.normalize_uri_options)
-      end
-
-      def to_s
-        "URI-normalizing #{super}"
-      end
-
-      def method_missing(*argv, &proc)
-        super(*argv, &proc)
-      end
-
-      def respond_to_missing?(method, private = false)
-        @frontier.respond_to?(method) || super
-      end
-    end
-  end
-end

data/lib/wayfarer/frontiers/redis_bloomfilter.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-require "bloomfilter-rb"
-
-module Wayfarer
-  module Frontiers
-    # A Redis bloomfilter.
-    # @api private
-    class RedisBloomfilter < MemoryBloomfilter
-      def initialize(config)
-        @conn = Redis.new(config.redis_opts)
-        @filter = BloomFilter::Redis.new(config.bloomfilter_opts.merge(db: @conn))
-        super(config)
-      end
-    end
-
-    # @override
-    def cache(*uris)
-      uris.each { |uri| @filter.insert(uri) }
-    end
-
-    # @override
-    def cached?(uri)
-      @filter.include?(uri)
-    end
-
-    # @override
-    def free
-      @filter.clear
-      @conn.disconnect!
-      super
-    end
-  end
-end

data/lib/wayfarer/frontiers/redis_frontier.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-require "redis"
-
-module Wayfarer
-  module Frontiers
-    # A Redis frontier
-    # @api private
-    class RedisFrontier < Frontier
-      def initialize(config)
-        @conn = Redis.new(config.redis_opts)
-        super(config)
-      end
-
-      # @override
-      def current_uris
-        @conn.smembers(current_uris_key).map { |str| URI(str) }
-      end
-
-      # @override
-      def staged_uris
-        @conn.smembers(staged_uris_key).map { |str| URI(str) }
-      end
-
-      # @override
-      def stage(*uris)
-        @conn.sadd(staged_uris_key, uris.map(&:to_s)) if uris.any?
-      end
-
-      # @override
-      def staged?(uri)
-        @conn.sismember(staged_uris_key, uri.to_s)
-      end
-
-      # @override
-      def cache(*uris)
-        @conn.sadd(cached_uris_key, uris.map(&:to_s)) if uris.any?
-      end
-
-      # @override
-      def cached?(uri)
-        @conn.sismember(cached_uris_key, uri.to_s)
-      end
-
-      # @override
-      def free
-        [current_uris_key, staged_uris_key, cached_uris_key].each do |key|
-          @conn.del(key)
-        end
-
-        @conn.disconnect!
-      end
-
-      private
-
-      def reset_staged_uris!
-        @conn.del(staged_uris_key)
-      end
-
-      # @override
-      def swap!
-        # Achieve: @current_uris = @staged_uris
-        @conn.rename(staged_uris_key, current_uris_key)
-      end
-
-      def filter_staged_uris!
-        @conn.sdiffstore(staged_uris_key, staged_uris_key, cached_uris_key)
-      end
-
-      def current_uris_key
-        "#{@config.uuid}_current_uris"
-      end
-
-      def staged_uris_key
-        "#{@config.uuid}_staged_uris"
-      end
-
-      def cached_uris_key
-        "#{@config.uuid}_cached_uris"
-      end
-    end
-  end
-end