wayfarer 0.0.3

data/lib/wayfarer/http_adapters/selenium_adapter.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "selenium-webdriver"
+require "selenium/emulated_features"
+require "capybara"
+
+module Wayfarer
+  module HTTPAdapters
+    # An adapter for Selenium WebDrivers
+    # @api private
+    class SeleniumAdapter
+      # @!attribute [r] driver
+      # @return [URI] the Selenium WebDriver.
+      attr_reader :driver
+
+      def initialize(config = Wayfarer.config)
+        @config = config
+      end
+
+      # Fetches a page.
+      # @return [Page]
+      def fetch(uri)
+        driver.navigate.to(uri)
+
+        Page.new(
+          uri: @driver.current_url,
+          status_code: @driver.response_code,
+          body: @driver.page_source,
+          headers: @driver.response_headers
+        )
+      end
+
+      # Closes the driver.
+      def reload!
+        @driver&.close
+        @driver = nil
+      end
+
+      # Quits the browser.
+      def free
+        @driver&.quit
+        @driver = nil
+      end
+
+      # The WebDriver.
+      def driver
+        @driver ||= instantiate_driver
+      end
+
+      # A Capybara driver that wraps the {#driver}.
+      # @see https://github.com/teamcapybara/capybara Capybara
+      def browser
+        @browser ||= instantiate_capybara_driver
+      end
+
+      private
+
+      def instantiate_driver
+        driver = Selenium::WebDriver.for(*@config.selenium_argv)
+        driver.manage.window.size = Selenium::WebDriver::Dimension.new(
+          *@config.window_size
+        )
+        driver
+      end
+
+      def instantiate_capybara_driver
+        Capybara.run_server = false
+        Capybara.current_driver = :selenium
+
+        capybara_driver = Capybara::Selenium::Driver.new(nil)
+        capybara_driver.instance_variable_set(:@browser, driver)
+
+        session = Capybara::Session.new(:selenium, nil)
+        session.instance_variable_set(:@driver, capybara_driver)
+
+        session
+      end
+    end
+  end
+end

data/lib/wayfarer/job.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "hooks"
+require "active_job"
+
+# TODO: I only want deep_dup
+require "active_support/all"
+
+module Wayfarer
+  # A {Job} is a class that has a {Routing::Router} with many {Routing::Rule}s
+  # which are matched against a URI. Rules map URIs onto job instance methods.
+  # Under the hood, jobs are instantiated within separate threads by a
+  # {Processor}. Every instance gets its own thread. If a URI is matched, its
+  # {Page} is retrieved, and made available to instance methods via {#page}.
+  #
+  # Jobs implement ActiveJob's Job API and are therefore compatible with a wide
+  # range of job queues. To run a job immediately, call ::perform_now.
+  # enqueue a job, call ::perform_later.
+  #
+  # @see https://github.com/rails/rails/tree/master/activejob rails/activejob
+  # @see http://edgeguides.rubyonrails.org/active_job_basics.html ActiveJob Basics
+  class Job < ActiveJob::Base
+    extend Forwardable
+
+    include Hooks
+    include Locals
+
+    # @!group Callbacks
+
+    # Callback that fires __once__ before any pages are retrieved.
+    # @method before_crawl
+    # @scope class
+    define_hook :before_crawl
+
+    # Callback that fires __once__ after all pages have been retrieved and
+    # processing is done.
+    # @method after_crawl
+    # @scope class
+    define_hook :after_crawl
+
+    # Callback that fires when HTTP adapters are instantiated.
+    # @method setup_adapter
+    # @scope class
+    # @yield [[HTTPAdapters::NetHTTPAdapter, HTTPAdapters::SeleniumAdapter], [Selenium::WebDriver::Driver, nil], [Capybara::Selenium::Driver, nil]]
+    define_hooks :setup_adapter
+
+    # @!endgroup
+
+    class << self
+      extend Forwardable
+
+      # @!attribute [w] router
+      attr_writer :router
+
+      # @!attribute [w] config
+      attr_writer :config
+
+      # Returns a class copy.
+      def prepare
+        duplicate = dup
+        duplicate.router = router.dup
+        duplicate.locals = locals.deep_dup
+        duplicate.config = config.dup
+
+        duplicate.locals.each do |(key, val)|
+          duplicate.locals[key] = Locals.thread_safe_counterpart(val)
+        end
+
+        duplicate.locals.each do |(key, _)|
+          duplicate.send(:define_method, key) do duplicate.locals[key] end
+          duplicate.send(:define_singleton_method, key) do
+            duplicate.locals[key]
+          end
+        end
+
+        duplicate
+      end
+
+      # A configuration based off the global {Wayfarer.config}.
+      # @yield [Configuration]
+      # @return [Configuration]
+      def config
+        @config ||= Wayfarer.config.clone
+        yield(@config) if block_given?
+        @config
+      end
+
+      # A router.
+      # If a block is passed in, it is evaluated within the {Router}'s instance.
+      # @return [Routing::Router]
+      def router(&proc)
+        @router ||= Routing::Router.new
+        @router.instance_eval(&proc) if block_given?
+        @router
+      end
+
+      alias route router
+      alias routes router
+
+      # Overshadows ActiveJob::Base's own logger
+      delegate logger: :config
+    end
+
+    # @!attribute [r] staged_uris
+    # @return [Array<String>, Array<URI>] URIs to stage for the next cycle.
+    # @see #stage
+    attr_reader :staged_uris
+
+    # @!attribute [rw] page
+    attr_writer :page
+
+    # @!attribute [rw] adapter
+    attr_accessor :adapter
+
+    # @!attribute [rw] params
+    attr_accessor :params
+
+    def initialize(*argv)
+      @halts = false
+      @staged_uris = []
+      super(*argv)
+    end
+
+    # Whether this job will stop processing.
+    def halts?
+      @halts
+    end
+
+    # Performs this job.
+    # @note ActiveJob API
+    # @override
+    def perform(*uris)
+      Crawl.new(self.class, *uris).execute
+    end
+
+    protected
+
+    # All following instance methods are available within actions.
+
+    # Sets a halting flag that signals the processor to stop its work.
+    def halt
+      @halts = true
+    end
+
+    # Adds URIs to process in the next cycle.
+    # If a relative path is given, an absolute URI is constructed from the
+    # current {#page}'s URI.
+    # @param [String, URI, Array<String>, Array<URI>]
+    def stage(*uris)
+      expanded = uris.flatten.map do |u|
+        if (uri = URI(u)).absolute?
+          uri
+        else
+          # URI#join would discard the path of page.uri.path
+          current = page.uri.dup
+          current.path = File.join(page.uri.path, uri.path)
+          current
+        end
+      end
+
+      # This method has somewhat become the guard keeper for invalid URIs that
+      # would lead to exceptions otherwise down the line
+      supported = expanded.select do |uri|
+        HTTPAdapters::NetHTTPAdapter::RECOGNIZED_URI_TYPES.any? do |type|
+          uri.is_a?(type)
+        end
+      end
+
+      @staged_uris.push(*supported)
+    end
+
+    # The {Page} representing the URI currently processed by an action.
+    # When using the Selenium adapter, {Page#body} gets refreshed on every call.
+    # Otherwise, subsequent DOM updates (i.e. JavaScript-induced) would be
+    # invisible.
+    # @return Page
+    def page
+      return @page unless self.class.config.http_adapter == :selenium
+
+      Page.new(
+        uri: @page.uri,
+        status_code: @page.uri,
+        body: driver.page_source,
+        headers: @page.headers
+      )
+    end
+
+    # The parsed response body.
+    # When using the Selenium adapter, this parses the body again on every call.
+    # Otherwise, subsequent DOM updates (i.e. JavaScript-induced) would be
+    # invisible.
+    # @method doc
+    # @see Page#doc
+    delegate doc: :page
+
+    # The Selenium WebDriver.
+    # @method driver
+    # @see https://github.com/peterc/pismo Pismo
+    # @see Page#driver
+    delegate driver: :adapter
+
+    # A Capybara driver that wraps the {#driver}.
+    # @method browser
+    # @see HTTPAdapters::SeleniumAdapter#browser
+    delegate browser: :adapter
+
+    # @method logger
+    delegate logger: :"self.class"
+  end
+end

data/lib/wayfarer/locals.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Wayfarer
+  # @api private
+  module Locals
+    def self.thread_safe_counterpart(value)
+      case value
+      when Array      then Concurrent::Array.new(value)
+      when Hash       then Concurrent::Hash[value]
+      when TrueClass  then Concurrent::AtomicBoolean.new(value)
+      when FalseClass then Concurrent::AtomicBoolean.new(value)
+      when Integer then Concurrent::AtomicFixnum.new(value)
+      else value
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      attr_reader :locals
+
+      def let(key)
+        raise "#let called without a block" unless block_given?
+        locals[key] = yield
+      end
+
+      def locals
+        @locals ||= {}
+      end
+
+      def locals=(locals)
+        @locals = locals
+      end
+    end
+  end
+end

data/lib/wayfarer/page.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "ostruct"
+require "forwardable"
+require "mime/types"
+require "mime-types"
+
+require "pismo" unless RUBY_PLATFORM == "java"
+
+module Wayfarer
+  # The representation of fetched pages
+  class Page
+    extend Forwardable
+
+    include Finders
+
+    # @!attribute [r] uri
+    # @return [URI] the URI of the page.
+    attr_reader :uri
+
+    # @!attribute [r] status_code
+    # @return [Fixnum] the response status code.
+    attr_reader :status_code
+
+    # @!attribute [r] body
+    # @return [String] the response body.
+    attr_accessor :body
+
+    # @!attribute [r] headers
+    # @return [Hash] the response headers.
+    attr_reader :headers
+
+    def initialize(attrs = {})
+      @uri         = attrs[:uri]
+      @status_code = attrs[:status_code]
+      @body        = attrs[:body]
+      @headers     = attrs[:headers]
+    end
+
+    # Returns a parsed representation of the fetched document depending on the
+    # Content-Type field.
+    # @return [OpenStruct] if the Content-Type field's sub-type is "json".
+    # @return [Nokogiri::XML::Document] if the Content-Type field's sub-type is "xml".
+    # @return [Nokogiri::HTML::Document] otherwise.
+    def doc
+      return @doc if @doc
+
+      # If no Content-Type field is present, assume HTML/XML
+      # TODO: Test
+      unless @headers["content-type"]
+        return @doc = Parsers::XMLParser.parse_html(@body)
+      end
+
+      content_type = @headers["content-type"].first
+      sub_type = MIME::Types[content_type].first.sub_type
+
+      # TODO: Tests
+      @doc = case sub_type
+             when "json"
+               Parsers::JSONParser.parse(@body)
+             when "xml"
+               Parsers::XMLParser.parse_xml(@body)
+             else
+               Parsers::XMLParser.parse_html(@body)
+             end
+    end
+
+    # Pismo is not supported on JRuby.
+    unless RUBY_PLATFORM == "java"
+      # `#images` is included from the Helpers module
+      # `#body` is an attribute reader defined above
+      delegate (Pismo::Document::ATTRIBUTE_METHODS - %i[images body]) => :pismo
+    end
+
+    private
+
+    # Returns a Pismo document.
+    # @note Not available on JRuby.
+    # @note Only succeeds when {#doc} returns a `Nokogiri::HTML::Document`.
+    # @return [Pismo::Document]
+    def pismo
+      @pismo_doc ||= instantiate_pismo_document
+    end
+
+    def instantiate_pismo_document
+      doc = Pismo::Document.allocate
+      doc.instance_variable_set(:@options, {})
+      doc.instance_variable_set(:@url, uri)
+      doc.instance_variable_set(:@html, body)
+      doc.instance_variable_set(:@doc, self.doc)
+      doc
+    end
+  end
+end

data/lib/wayfarer/parsers/json_parser.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "oj" unless RUBY_PLATFORM == "java"
+
+module Wayfarer
+  module Parsers
+    # A wrapper class for parsing JSON.
+    # @private
+    module JSONParser
+      module_function
+
+      # Parses a JSON string.
+      # @param [String] json_str the JSON string to parse.
+      # @return [OpenStruct]
+      def parse(json_str)
+        RUBY_PLATFORM == "java" ? JSON.parse(json_str) : Oj.load(json_str)
+      end
+    end
+  end
+end

data/lib/wayfarer/parsers/xml_parser.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Wayfarer
+  module Parsers
+    # A wrapper class for parsing HTML/XML.
+    # @private
+    module XMLParser
+      module_function
+
+      # Parses an XML string.
+      # @param [String] xml_str the XML string to parse.
+      # @return [Nokogiri::XML::Document]
+      def parse_xml(xml_str)
+        Nokogiri::XML(xml_str)
+      end
+
+      # Parses a HTML string.
+      # @param [String] html_str the HTML string to parse.
+      # @return [Nokogiri::HTML::Document]
+      def parse_html(html_str)
+        Nokogiri::HTML(html_str)
+      end
+    end
+  end
+end

data/lib/wayfarer/processor.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "pp"
+require "concurrent"
+require "observer"
+
+module Wayfarer
+  # Runs jobs.
+  class Processor
+    extend Forwardable
+
+    include Observable
+    include CrawlObserver::Events
+    include CrawlObserver::ObservableShortcuts
+
+    attr_reader :job
+
+    delegate config: :job
+    delegate logger: :config
+
+    def initialize(job, frontier, dispatcher)
+      @job = job
+      @frontier = frontier
+      @dispatcher = dispatcher
+      @halted = Concurrent::AtomicBoolean.new(false)
+    end
+
+    # Whether processing is done.
+    # @return [true, false]
+    def halted?
+      @halted.value
+    end
+
+    # Sets a halt flag.
+    def halt!
+      @halted.make_true
+    end
+
+    # Runs the job.
+    # @param [*Array<URI>, *Array<String>] uris
+    def run(*_uris)
+      notify_observers!(FirstCycle.new(@frontier))
+
+      while @halted.false? && @frontier.cycle
+        current_uris = @frontier.current_uris
+        queue = current_uris.inject(Queue.new, :push)
+
+        notify_observers!(NewCycle.new(current_uris.count))
+
+        @threads = Array.new(config.connection_count) do
+          Thread.new do
+            begin
+              loop do
+                uri = queue.pop(true)
+                break if uri.nil? || @halted.true?
+                handle_dispatch_result(@dispatcher.dispatch(@job, uri))
+              end
+            rescue ThreadError
+              notify_observers!(CycleFinished.new)
+            end
+          end
+        end
+
+        @threads.each(&:join)
+
+        notify_observers!(AboutToCycle.new(@frontier.staged_uris.count))
+      end
+    ensure
+      halt!
+      @frontier.free
+      @dispatcher.adapter_pool.free
+    end
+
+    private
+
+    def handle_dispatch_result(result)
+      case result
+      when Dispatcher::Mismatch then handle_mismatch(result)
+      when Dispatcher::Halt     then handle_halt(result)
+      when Dispatcher::Stage    then handle_stage(result)
+      when Dispatcher::Error    then handle_error(result)
+      end
+    end
+
+    def handle_mismatch(mismatch)
+      notify_observers!(MismatchedURI.new(mismatch.uri))
+    end
+
+    def handle_halt(halt)
+      notify_observers!(HaltInitiated.new(halt.action, halt.uri))
+      halt!
+    end
+
+    def handle_stage(stage)
+      notify_observers!(StagingURIs.new(stage.uris.count))
+      @frontier.stage(*stage.uris) unless halted?
+    end
+
+    def handle_error(error)
+      notify_observers!(UnhandledError.new(error.exception))
+    end
+  end
+end

data/lib/wayfarer/routing/custom_rule.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Wayfarer
+  module Routing
+    # @private
+    class CustomRule < Rule
+      def initialize(delegate_or_block = proc, opts = {}, &proc)
+        @delegate_or_block = delegate_or_block
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        !!@delegate_or_block.call(uri)
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/filetypes_rule.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+require "uri"
+
+module Wayfarer
+  module Routing
+    # @private
+    class FiletypesRule < Rule
+      def initialize(types, opts = {}, &proc)
+        @types = types
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        @types.any? { |type| uri.path =~ /\.#{type}$/ }
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/host_rule.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    # @private
+    class HostRule < Rule
+      def initialize(str_or_regexp, opts = {}, &proc)
+        @str_or_regexp = str_or_regexp
+        super(opts, &proc)
+      end
+
+      # rubocop:disable Style/CaseEquality
+      def match!(uri)
+        @str_or_regexp === uri.host
+      end
+      # rubocop:enable Style/CaseEquality
+    end
+  end
+end

data/lib/wayfarer/routing/path_rule.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "mustermann"
+
+module Wayfarer
+  module Routing
+    # @private
+    class PathRule < Rule
+      attr_reader :matcher
+
+      def initialize(arg, opts = {}, &proc)
+        @matcher = if arg.is_a? String
+                     Mustermann.new(arg, type: Wayfarer.config.mustermann_type)
+                   else
+                     arg
+                   end
+
+        super(opts, &proc)
+      end
+
+      def params(uri)
+        return {} unless match!(uri)
+
+        path = uri.path
+
+        if @matcher.is_a? Mustermann
+          @matcher.params(path)
+        else
+          captures = @matcher.match(full_path(uri)).captures
+
+          captures.each.with_index.reduce({}) do |hash, (capture, i)|
+            hash.merge(i.to_s => capture)
+          end
+        end
+      end
+
+      private
+
+      # rubocop:disable Style/CaseEquality
+      def match!(uri)
+        if @matcher.is_a? Mustermann
+          @matcher === uri.path
+        else
+          @matcher =~ full_path(uri)
+        end
+      end
+      # rubocop:enable Style/CaseEquality
+
+      def full_path(uri)
+        "#{uri.path}?#{uri.query}##{uri.fragment}"
+      end
+    end
+  end
+end