wayfarer-jruby 0.0.1

data/lib/wayfarer/frontiers/redis_bloomfilter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "bloomfilter-rb"
+
+module Wayfarer
+  module Frontiers
+    # A Redis bloomfilter.
+    # @api private
+    class RedisBloomfilter < MemoryBloomfilter
+      def initialize(config)
+        @conn = Redis.new(config.redis_opts)
+        @filter = BloomFilter::Redis.new(config.bloomfilter_opts.merge(db: @conn))
+        super(config)
+      end
+    end
+
+    # @override
+    def cache(*uris)
+      uris.each { |uri| @filter.insert(uri) }
+    end
+
+    # @override
+    def cached?(uri)
+      @filter.include?(uri)
+    end
+
+    # @override
+    def free
+      @filter.clear
+      @conn.disconnect!
+      super
+    end
+  end
+end

data/lib/wayfarer/frontiers/redis_frontier.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "redis"
+
+module Wayfarer
+  module Frontiers
+    # A Redis frontier
+    # @api private
+    class RedisFrontier < Frontier
+      def initialize(config)
+        @conn = Redis.new(config.redis_opts)
+        super(config)
+      end
+
+      # @override
+      def current_uris
+        @conn.smembers(current_uris_key).map { |str| URI(str) }
+      end
+
+      # @override
+      def staged_uris
+        @conn.smembers(staged_uris_key).map { |str| URI(str) }
+      end
+
+      # @override
+      def stage(*uris)
+        @conn.sadd(staged_uris_key, uris.map(&:to_s)) if uris.any?
+      end
+
+      # @override
+      def staged?(uri)
+        @conn.sismember(staged_uris_key, uri.to_s)
+      end
+
+      # @override
+      def cache(*uris)
+        @conn.sadd(cached_uris_key, uris.map(&:to_s)) if uris.any?
+      end
+
+      # @override
+      def cached?(uri)
+        @conn.sismember(cached_uris_key, uri.to_s)
+      end
+
+      # @override
+      def free
+        [current_uris_key, staged_uris_key, cached_uris_key].each do |key|
+          @conn.del(key)
+        end
+
+        @conn.disconnect!
+      end
+
+      private
+
+      def reset_staged_uris!
+        @conn.del(staged_uris_key)
+      end
+
+      # @override
+      def swap!
+        # Achieve: @current_uris = @staged_uris
+        @conn.rename(staged_uris_key, current_uris_key)
+      end
+
+      def filter_staged_uris!
+        @conn.sdiffstore(staged_uris_key, staged_uris_key, cached_uris_key)
+      end
+
+      def current_uris_key
+        "#{@config.uuid}_current_uris"
+      end
+
+      def staged_uris_key
+        "#{@config.uuid}_staged_uris"
+      end
+
+      def cached_uris_key
+        "#{@config.uuid}_cached_uris"
+      end
+    end
+  end
+end

data/lib/wayfarer/http_adapters/adapter_pool.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "connection_pool"
+
+module Wayfarer
+  module HTTPAdapters
+    # A connection pool that hands out HTTP adapters.
+    # @private
+    class AdapterPool
+      extend Forwardable
+
+      def initialize(job)
+        @job = job
+        @config = job.config
+
+        size = @config.connection_count
+        timeout = @config.connection_timeout
+
+        @pool = ConnectionPool.new(
+          size: size,
+          timeout: timeout,
+          &method(:instantiate_adapter)
+        )
+      end
+
+      # Shuts down all HTTP adapters
+      def free
+        @pool.shutdown(&:free)
+      end
+
+      private
+
+      def instantiate_adapter
+        adapter = if @config.http_adapter == :selenium
+                    HTTPAdapters::SeleniumAdapter.new(@config)
+                  else
+                    HTTPAdapters::NetHTTPAdapter.instance(@config)
+                  end
+
+        @job.run_hook(
+          :setup_adapter,
+          adapter,
+          adapter.try(:driver),
+          adapter.try(:browser)
+        )
+
+        adapter
+      end
+
+      def method_missing(method, *argv, &proc)
+        super if method == :shutdown
+        @pool.public_send(method, *argv, &proc)
+      end
+
+      def respond_to_missing?(method, private = false)
+        return false if method == :shutdown
+        @pool.respond_to?(method) || super
+      end
+    end
+  end
+end

data/lib/wayfarer/http_adapters/net_http_adapter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "net/http"
+require "net/http/persistent"
+
+module Wayfarer
+  module HTTPAdapters
+    # A singleton adapter for net-http-persistent.
+    # @api private
+    class NetHTTPAdapter
+      # Supported standard lib classes
+      RECOGNIZED_URI_TYPES = [
+        URI::HTTP,
+        URI::HTTPS
+      ].freeze
+
+      MalformedURI = Class.new(StandardError)
+      MalformedRedirectURI = Class.new(StandardError)
+      MaximumRedirectCountReached = Class.new(StandardError)
+
+      attr_accessor :request_header_overrides
+
+      # TODO: Remove default parameter value
+      def self.instance(config = Wayfarer.config)
+        @@instance ||= new(config)
+      end
+
+      def initialize(config)
+        @config = config
+        @conn = Net::HTTP::Persistent.new("wayfarer-#{SecureRandom.uuid}")
+      end
+
+      # This is a singleton class. Use ::instance instead.
+      private_class_method :new
+
+      # Fetches a page.
+      # @return [Page]
+      # @raise [MalformedURI] if the URI is not supported.
+      # @raise [MalformedRedirectURI] if a redirection URI is not supported.
+      # @raise [MaximumRedirectCountReached] if too many redirections are
+      # encountered.
+      def fetch(uri, redirects_followed = 0)
+        if !RECOGNIZED_URI_TYPES.include?(uri.class)
+          raise _ = if redirects_followed.positive?
+                      MalformedRedirectURI
+                    else
+                      MalformedURI
+                    end
+        elsif redirects_followed > @config.max_http_redirects
+          raise MaximumRedirectCountReached
+        end
+
+        res = @conn.request(uri)
+
+        if res.is_a? Net::HTTPRedirection
+          redirect_uri = URI(res["location"])
+          return fetch(redirect_uri, redirects_followed + 1)
+        end
+
+        Page.new(
+          uri: uri,
+          status_code: res.code.to_i,
+          body: res.body,
+          headers: res.to_hash
+        )
+      rescue SocketError
+        raise MalformedURI
+      end
+
+      # Shuts down all connections.
+      def free
+        @conn.shutdown
+      end
+    end
+  end
+end

data/lib/wayfarer/http_adapters/selenium_adapter.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "selenium-webdriver"
+require "selenium/emulated_features"
+require "capybara"
+
+module Wayfarer
+  module HTTPAdapters
+    # An adapter for Selenium WebDrivers
+    # @api private
+    class SeleniumAdapter
+      # @!attribute [r] driver
+      # @return [URI] the Selenium WebDriver.
+      attr_reader :driver
+
+      def initialize(config = Wayfarer.config)
+        @config = config
+      end
+
+      # Fetches a page.
+      # @return [Page]
+      def fetch(uri)
+        driver.navigate.to(uri)
+
+        Page.new(
+          uri: @driver.current_url,
+          status_code: @driver.response_code,
+          body: @driver.page_source,
+          headers: @driver.response_headers
+        )
+      end
+
+      # Closes the driver.
+      def reload!
+        @driver&.close
+        @driver = nil
+      end
+
+      # Quits the browser.
+      def free
+        @driver&.quit
+        @driver = nil
+      end
+
+      # The WebDriver.
+      def driver
+        @driver ||= instantiate_driver
+      end
+
+      # A Capybara driver that wraps the {#driver}.
+      # @see https://github.com/teamcapybara/capybara Capybara
+      def browser
+        @browser ||= instantiate_capybara_driver
+      end
+
+      private
+
+      def instantiate_driver
+        driver = Selenium::WebDriver.for(*@config.selenium_argv)
+        driver.manage.window.size = Selenium::WebDriver::Dimension.new(
+          *@config.window_size
+        )
+        driver
+      end
+
+      def instantiate_capybara_driver
+        Capybara.run_server = false
+        Capybara.current_driver = :selenium
+
+        capybara_driver = Capybara::Selenium::Driver.new(nil)
+        capybara_driver.instance_variable_set(:@browser, driver)
+
+        session = Capybara::Session.new(:selenium, nil)
+        session.instance_variable_set(:@driver, capybara_driver)
+
+        session
+      end
+    end
+  end
+end

data/lib/wayfarer/job.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "hooks"
+require "active_job"
+
+# TODO: I only want deep_dup
+require "active_support/all"
+
+module Wayfarer
+  # A {Job} is a class that has a {Routing::Router} with many {Routing::Rule}s
+  # which are matched against a URI. Rules map URIs onto job instance methods.
+  # Under the hood, jobs are instantiated within separate threads by a
+  # {Processor}. Every instance gets its own thread. If a URI is matched, its
+  # {Page} is retrieved, and made available to instance methods via {#page}.
+  #
+  # Jobs implement ActiveJob's Job API and are therefore compatible with a wide
+  # range of job queues. To run a job immediately, call ::perform_now.
+  # enqueue a job, call ::perform_later.
+  #
+  # @see https://github.com/rails/rails/tree/master/activejob rails/activejob
+  # @see http://edgeguides.rubyonrails.org/active_job_basics.html ActiveJob Basics
+  class Job < ActiveJob::Base
+    extend Forwardable
+
+    include Hooks
+    include Locals
+
+    # @!group Callbacks
+
+    # Callback that fires __once__ before any pages are retrieved.
+    # @method before_crawl
+    # @scope class
+    define_hook :before_crawl
+
+    # Callback that fires __once__ after all pages have been retrieved and
+    # processing is done.
+    # @method after_crawl
+    # @scope class
+    define_hook :after_crawl
+
+    # Callback that fires when HTTP adapters are instantiated.
+    # @method setup_adapter
+    # @scope class
+    # @yield [[HTTPAdapters::NetHTTPAdapter, HTTPAdapters::SeleniumAdapter], [Selenium::WebDriver::Driver, nil], [Capybara::Selenium::Driver, nil]]
+    define_hooks :setup_adapter
+
+    # @!endgroup
+
+    class << self
+      extend Forwardable
+
+      # @!attribute [w] router
+      attr_writer :router
+
+      # @!attribute [w] config
+      attr_writer :config
+
+      # Returns a class copy.
+      def prepare
+        duplicate = dup
+        duplicate.router = router.dup
+        duplicate.locals = locals.deep_dup
+        duplicate.config = config.dup
+
+        duplicate.locals.each do |(key, val)|
+          duplicate.locals[key] = Locals.thread_safe_counterpart(val)
+        end
+
+        duplicate.locals.each do |(key, _)|
+          duplicate.send(:define_method, key) do duplicate.locals[key] end
+          duplicate.send(:define_singleton_method, key) do
+            duplicate.locals[key]
+          end
+        end
+
+        duplicate
+      end
+
+      # A configuration based off the global {Wayfarer.config}.
+      # @yield [Configuration]
+      # @return [Configuration]
+      def config
+        @config ||= Wayfarer.config.clone
+        yield(@config) if block_given?
+        @config
+      end
+
+      # A router.
+      # If a block is passed in, it is evaluated within the {Router}'s instance.
+      # @return [Routing::Router]
+      def router(&proc)
+        @router ||= Routing::Router.new
+        @router.instance_eval(&proc) if block_given?
+        @router
+      end
+
+      alias route router
+      alias routes router
+
+      # Overshadows ActiveJob::Base's own logger
+      delegate logger: :config
+    end
+
+    # @!attribute [r] staged_uris
+    # @return [Array<String>, Array<URI>] URIs to stage for the next cycle.
+    # @see #stage
+    attr_reader :staged_uris
+
+    # @!attribute [rw] page
+    attr_writer :page
+
+    # @!attribute [rw] adapter
+    attr_accessor :adapter
+
+    # @!attribute [rw] params
+    attr_accessor :params
+
+    def initialize(*argv)
+      @halts = false
+      @staged_uris = []
+      super(*argv)
+    end
+
+    # Whether this job will stop processing.
+    def halts?
+      @halts
+    end
+
+    # Performs this job.
+    # @note ActiveJob API
+    # @override
+    def perform(*uris)
+      Crawl.new(self.class, *uris).execute
+    end
+
+    protected
+
+    # All following instance methods are available within actions.
+
+    # Sets a halting flag that signals the processor to stop its work.
+    def halt
+      @halts = true
+    end
+
+    # Adds URIs to process in the next cycle.
+    # If a relative URI is given, the page's protocol and hostname get
+    # prepended.
+    # @param [String, URI, Array<String>, Array<URI>]
+    def stage(*uris)
+      @staged_uris.push(*uris.flatten)
+    end
+
+    # The {Page} representing the URI currently processed by an action.
+    # When using the Selenium adapter, {Page#body} gets refreshed on every call.
+    # Otherwise, subsequent DOM updates (i.e. JavaScript-induced) would be
+    # invisible.
+    # @return Page
+    def page
+      return @page unless self.class.config.http_adapter == :selenium
+
+      Page.new(
+        uri: @page.uri,
+        status_code: @page.uri,
+        body: driver.page_source,
+        headers: @page.headers
+      )
+    end
+
+    # The parsed response body.
+    # When using the Selenium adapter, this parses the body again on every call.
+    # Otherwise, subsequent DOM updates (i.e. JavaScript-induced) would be
+    # invisible.
+    # @method doc
+    # @see Page#doc
+    delegate doc: :page
+
+    # The Selenium WebDriver.
+    # @method driver
+    # @see https://github.com/peterc/pismo Pismo
+    # @see Page#driver
+    delegate driver: :adapter
+
+    # A Capybara driver that wraps the {#driver}.
+    # @method browser
+    # @see HTTPAdapters::SeleniumAdapter#browser
+    delegate browser: :adapter
+
+    # @method logger
+    delegate logger: :"self.class"
+  end
+end