wayfarer-jruby 0.0.1

data/lib/wayfarer/locals.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+module Wayfarer
+  # @api private
+  module Locals
+    def self.thread_safe_counterpart(value)
+      case value
+      when Array      then Concurrent::Array.new(value)
+      when Hash       then Concurrent::Hash[value]
+      when TrueClass  then Concurrent::AtomicBoolean.new(value)
+      when FalseClass then Concurrent::AtomicBoolean.new(value)
+      when Integer then Concurrent::AtomicFixnum.new(value)
+      else value
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      attr_reader :locals
+
+      def let(key)
+        raise "#let called without a block" unless block_given?
+        locals[key] = yield
+      end
+
+      def locals
+        @locals ||= {}
+      end
+
+      def locals=(locals)
+        @locals = locals
+      end
+    end
+  end
+end

data/lib/wayfarer/page.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "ostruct"
+require "forwardable"
+require "mime/types"
+require "mime-types"
+
+require "pismo" unless RUBY_PLATFORM == "java"
+
+module Wayfarer
+  # The representation of fetched pages
+  class Page
+    extend Forwardable
+
+    include Finders
+
+    # @!attribute [r] uri
+    # @return [URI] the URI of the page.
+    attr_reader :uri
+
+    # @!attribute [r] status_code
+    # @return [Fixnum] the response status code.
+    attr_reader :status_code
+
+    # @!attribute [r] body
+    # @return [String] the response body.
+    attr_accessor :body
+
+    # @!attribute [r] headers
+    # @return [Hash] the response headers.
+    attr_reader :headers
+
+    def initialize(attrs = {})
+      @uri         = attrs[:uri]
+      @status_code = attrs[:status_code]
+      @body        = attrs[:body]
+      @headers     = attrs[:headers]
+    end
+
+    # Returns a parsed representation of the fetched document depending on the
+    # Content-Type field.
+    # @return [OpenStruct] if the Content-Type field's sub-type is "json".
+    # @return [Nokogiri::XML::Document] if the Content-Type field's sub-type is "xml".
+    # @return [Nokogiri::HTML::Document] otherwise.
+    def doc
+      return @doc if @doc
+
+      # If no Content-Type field is present, assume HTML/XML
+      # TODO: Test
+      unless @headers["content-type"]
+        return @doc = Parsers::XMLParser.parse_html(@body)
+      end
+
+      content_type = @headers["content-type"].first
+      sub_type = MIME::Types[content_type].first.sub_type
+
+      # TODO: Tests
+      @doc = case sub_type
+             when "json"
+               Parsers::JSONParser.parse(@body)
+             when "xml"
+               Parsers::XMLParser.parse_xml(@body)
+             else
+               Parsers::XMLParser.parse_html(@body)
+             end
+    end
+
+    # Pismo is not supported on JRuby.
+    unless RUBY_PLATFORM == "java"
+      # `#images` is included from the Helpers module
+      # `#body` is an attribute reader defined above
+      delegate (Pismo::Document::ATTRIBUTE_METHODS - %i[images body]) => :pismo
+    end
+
+    private
+
+    # Returns a Pismo document.
+    # @note Not available on JRuby.
+    # @note Only succeeds when {#doc} returns a `Nokogiri::HTML::Document`.
+    # @return [Pismo::Document]
+    def pismo
+      @pismo_doc ||= instantiate_pismo_document
+    end
+
+    def instantiate_pismo_document
+      doc = Pismo::Document.allocate
+      doc.instance_variable_set(:@options, {})
+      doc.instance_variable_set(:@url, uri)
+      doc.instance_variable_set(:@html, body)
+      doc.instance_variable_set(:@doc, self.doc)
+      doc
+    end
+  end
+end

data/lib/wayfarer/parsers/json_parser.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "oj" unless RUBY_PLATFORM == "java"
+
+module Wayfarer
+  module Parsers
+    # A wrapper class for parsing JSON.
+    # @private
+    module JSONParser
+      module_function
+
+      # Parses a JSON string.
+      # @param [String] json_str the JSON string to parse.
+      # @return [OpenStruct]
+      def parse(json_str)
+        RUBY_PLATFORM == "java" ? JSON.parse(json_str) : Oj.load(json_str)
+      end
+    end
+  end
+end

data/lib/wayfarer/parsers/xml_parser.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Wayfarer
+  module Parsers
+    # A wrapper class for parsing HTML/XML.
+    # @private
+    module XMLParser
+      module_function
+
+      # Parses an XML string.
+      # @param [String] xml_str the XML string to parse.
+      # @return [Nokogiri::XML::Document]
+      def parse_xml(xml_str)
+        Nokogiri::XML(xml_str)
+      end
+
+      # Parses a HTML string.
+      # @param [String] html_str the HTML string to parse.
+      # @return [Nokogiri::HTML::Document]
+      def parse_html(html_str)
+        Nokogiri::HTML(html_str)
+      end
+    end
+  end
+end

data/lib/wayfarer/processor.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "pp"
+require "concurrent"
+require "observer"
+
+module Wayfarer
+  # Runs jobs.
+  class Processor
+    extend Forwardable
+
+    include Observable
+    include CrawlObserver::Events
+    include CrawlObserver::ObservableShortcuts
+
+    attr_reader :job
+
+    delegate config: :job
+    delegate logger: :config
+
+    def initialize(job, frontier, dispatcher)
+      @job = job
+      @frontier = frontier
+      @dispatcher = dispatcher
+      @halted = Concurrent::AtomicBoolean.new(false)
+    end
+
+    # Whether processing is done.
+    # @return [true, false]
+    def halted?
+      @halted.value
+    end
+
+    # Sets a halt flag.
+    def halt!
+      @halted.make_true
+    end
+
+    # Runs the job.
+    # @param [*Array<URI>, *Array<String>] uris
+    def run(*_uris)
+      notify_observers!(FirstCycle.new(@frontier))
+
+      while @halted.false? && @frontier.cycle
+        current_uris = @frontier.current_uris
+        queue = current_uris.inject(Queue.new, :push)
+
+        notify_observers!(NewCycle.new(current_uris.count))
+
+        @threads = Array.new(config.connection_count) do
+          Thread.new do
+            begin
+              loop do
+                uri = queue.pop(true)
+                break if uri.nil? || @halted.true?
+                handle_dispatch_result(@dispatcher.dispatch(@job, uri))
+              end
+            rescue ThreadError
+              notify_observers!(CycleFinished.new)
+            end
+          end
+        end
+
+        @threads.each(&:join)
+
+        notify_observers!(AboutToCycle.new(@frontier.staged_uris.count))
+      end
+    ensure
+      halt!
+      @frontier.free
+      @dispatcher.adapter_pool.free
+    end
+
+    private
+
+    def handle_dispatch_result(result)
+      case result
+      when Dispatcher::Mismatch then handle_mismatch(result)
+      when Dispatcher::Halt     then handle_halt(result)
+      when Dispatcher::Stage    then handle_stage(result)
+      when Dispatcher::Error    then handle_error(result)
+      end
+    end
+
+    def handle_mismatch(mismatch)
+      notify_observers!(MismatchedURI.new(mismatch.uri))
+    end
+
+    def handle_halt(halt)
+      notify_observers!(HaltInitiated.new(halt.action, halt.uri))
+      halt!
+    end
+
+    def handle_stage(stage)
+      notify_observers!(StagingURIs.new(stage.uris.count))
+      @frontier.stage(*stage.uris) unless halted?
+    end
+
+    def handle_error(error)
+      notify_observers!(UnhandledError.new(error.exception))
+    end
+  end
+end

data/lib/wayfarer/routing/host_rule.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    # @private
+    class HostRule < Rule
+      def initialize(str_or_regexp, opts = {}, &proc)
+        @str_or_regexp = str_or_regexp
+        super(opts, &proc)
+      end
+
+      # rubocop:disable Style/CaseEquality
+      def match!(uri)
+        @str_or_regexp === uri.host
+      end
+      # rubocop:enable Style/CaseEquality
+    end
+  end
+end

data/lib/wayfarer/routing/path_rule.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "mustermann"
+
+module Wayfarer
+  module Routing
+    # @private
+    class PathRule < Rule
+      attr_reader :matcher
+
+      def initialize(arg, opts = {}, &proc)
+        @matcher = if arg.is_a? String
+                     Mustermann.new(arg, type: Wayfarer.config.mustermann_type)
+                   else
+                     arg
+                   end
+
+        super(opts, &proc)
+      end
+
+      def params(uri)
+        return {} unless match!(uri)
+
+        path = uri.path
+
+        if @matcher.is_a? Mustermann
+          @matcher.params(path)
+        else
+          captures = @matcher.match(full_path(uri)).captures
+
+          captures.each.with_index.reduce({}) do |hash, (capture, i)|
+            hash.merge(i.to_s => capture)
+          end
+        end
+      end
+
+      private
+
+      # rubocop:disable Style/CaseEquality
+      def match!(uri)
+        if @matcher.is_a? Mustermann
+          @matcher === uri.path
+        else
+          @matcher =~ full_path(uri)
+        end
+      end
+      # rubocop:enable Style/CaseEquality
+
+      def full_path(uri)
+        "#{uri.path}?#{uri.query}##{uri.fragment}"
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/query_rule.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+module Wayfarer
+  module Routing
+    # @private
+    class QueryRule < Rule
+      def initialize(field_constraints, opts = {}, &proc)
+        @field_constraints = field_constraints
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        CGI.parse(uri.query).none? do |field, vals| violates?(field, vals) end
+      rescue NoMethodError
+        # `CGI::parse` throws a `NoMethodError` if `uri.query` is an empty
+        # string
+        false
+      end
+
+      # rubocop:disable Lint/AssignmentInCondition
+      def violates?(field, vals)
+        return false unless constraint = @field_constraints[field.to_sym]
+        violates_constraint?(constraint, vals)
+      end
+      # rubocop:enable Lint/AssignmentInCondition
+
+      def violates_constraint?(constraint, vals)
+        case constraint
+        when String  then violates_string?(constraint, vals)
+        when Integer then violates_integer?(constraint, vals)
+        when Regexp  then violates_regexp?(constraint, vals)
+        when Range   then violates_range?(constraint, vals)
+        end
+      end
+
+      def violates_string?(str, vals)
+        vals.none? { |val| str == val }
+      end
+
+      def violates_integer?(int, vals)
+        vals.none? do |val| int == Integer(val) end
+      rescue ArgumentError
+        true
+      end
+
+      def violates_regexp?(regexp, vals)
+        vals.none? { |val| regexp.match(val) }
+      end
+
+      def violates_range?(range, vals)
+        vals.none? { |val| range.include?(val.to_i) }
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/router.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Wayfarer
+  module Routing
+    # A {Router} maps URIs onto a {Job}'s instance methods.
+    class Router
+      extend Forwardable
+
+      # @!attribute [r] rule
+      # @return [Rule]
+      attr_reader :rule
+
+      # @!attribute [r] blacklist
+      # @return [Rule]
+      attr_reader :blacklist
+
+      def initialize
+        @rule = Rule.new
+        @blacklist = Rule.new
+      end
+
+      delegate %i[
+        uri
+        host
+        path
+        query
+      ] => :rule
+
+      # Returns the associated instance method (action) of the first rule that
+      # matches a URI and the collected parameter hash from the rule chain.
+      # @return [[Boolean, Symbol, Hash]] if a matching rule exists.
+      # @return [false] if no matching rule exists or the URI is forbidden.
+      def route(uri)
+        return false if forbids?(uri)
+
+        # TODO: Use structs instead
+        is_matching, params, action = @rule.invoke(uri)
+        return action, params if is_matching && params
+
+        false
+      end
+
+      # Whether a route matches the URI.
+      # TODO: Test
+      def routes?(uri)
+        !!route(uri)
+      end
+
+      # Adds a {Rule} to the blacklist.
+      def forbid(opts = {}, &proc)
+        @blacklist.build_child_rule_chain_from_options(opts)
+        @blacklist.instance_eval(&proc) if block_given?
+        @blacklist
+      end
+
+      # Whether the URI is matched by the blacklist rule.
+      # @see #forbid
+      def forbids?(uri)
+        @blacklist.matches?(uri)
+      end
+
+      # Whether the URI is allowed.
+      # @see #forbid
+      def allows?(uri)
+        !forbids?(uri)
+      end
+    end
+  end
+end