wayfarer-jruby 0.0.1

data/docs/routing/uri_rules.md

@@ -0,0 +1,18 @@
+---
+layout: default
+title: URI Rules
+---
+
+# URI rules
+
+URI rules match against a string.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.uri "https://example.com"
+end
+{% endhighlight %}
+
+Matches:
+
+* Only `https://example.com`

data/examples/collect_github_issues.rb

@@ -0,0 +1,65 @@
+require_relative "../lib/wayfarer"
+
+class CollectGithubIssues < Wayfarer::Job
+  config.connection_count = 4
+  config.logger.level = :fatal
+
+  let(:records) { [] }
+
+  routes do
+    host "github.com" do
+      path "/:user/:repo",            to: :repository
+      path "/:user/:repo/issues",     to: :index
+      path "/:user/:repo/issues/:id", to: :show
+    end
+  end
+
+  after_crawl do
+    records.each do |issue|
+      # Save them somewhere?
+      puts issue
+    end
+  end
+
+  def repository
+    stage navigation_links
+  end
+
+  def index
+    stage issue_listing_links, next_page
+  end
+
+  def show
+    return halt if records.count > 30
+
+    records << {
+      id: params[:id],
+      title: issue_title,
+      author: issue_author
+    }
+  end
+
+  private
+
+  def issue_title
+    doc.css(".js-issue-title").text.strip
+  end
+
+  def issue_author
+    doc.css(".TableObject-item .author").text.strip
+  end
+
+  def navigation_links
+    page.links ".reponav-item"
+  end
+
+  def issue_listing_links
+    page.links ".issues-listing"
+  end
+
+  def next_page
+    page.links ".next_page"
+  end
+end
+
+CollectGithubIssues.perform_now("https://github.com/rails/rails")

data/examples/find_foobar_on_wikipedia.rb

@@ -0,0 +1,23 @@
+require_relative "../lib/wayfarer"
+
+class FindFoobarOnWikipedia < Wayfarer::Job
+  config.http_adapter = :selenium
+  config.selenium_argv = [:chrome]
+  config.connection_count = 4
+
+  let(:keywords) { [] }
+
+  route.host "en.wikipedia.org", to: :article
+
+  def article
+    if page.body =~ /Foobar/
+      driver.save_screenshot("/tmp/foobar.png")
+      return halt
+    end
+
+    keywords << page.keywords
+    stage page.links
+  end
+end
+
+FindFoobarOnWikipedia.perform_now("https://en.wikipedia.org/wiki/Special:Random")

data/lib/wayfarer.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/Documentation
+require "logger"
+require "uri"
+
+# Plumbing
+require_relative "wayfarer/configuration"
+
+# Routing
+require_relative "wayfarer/routing/rule"
+require_relative "wayfarer/routing/uri_rule"
+require_relative "wayfarer/routing/host_rule"
+require_relative "wayfarer/routing/path_rule"
+require_relative "wayfarer/routing/query_rule"
+require_relative "wayfarer/routing/router"
+
+# Networking
+require_relative "wayfarer/http_adapters/net_http_adapter"
+require_relative "wayfarer/http_adapters/selenium_adapter"
+require_relative "wayfarer/http_adapters/adapter_pool"
+
+# Parsers
+require_relative "wayfarer/parsers/xml_parser"
+require_relative "wayfarer/parsers/json_parser"
+
+# Frontiers
+require_relative "wayfarer/frontiers/frontier"
+require_relative "wayfarer/frontiers/memory_frontier"
+require_relative "wayfarer/frontiers/redis_frontier"
+require_relative "wayfarer/frontiers/normalize_uris"
+
+unless RUBY_PLATFORM == "java"
+  require_relative "wayfarer/frontiers/memory_trie_frontier"
+  require_relative "wayfarer/frontiers/memory_bloomfilter"
+  require_relative "wayfarer/frontiers/redis_bloomfilter"
+end
+
+# Processing
+require_relative "wayfarer/crawl"
+require_relative "wayfarer/crawl_observer"
+require_relative "wayfarer/locals"
+require_relative "wayfarer/job"
+require_relative "wayfarer/finders"
+require_relative "wayfarer/page"
+require_relative "wayfarer/dispatcher"
+require_relative "wayfarer/processor"
+
+module Wayfarer
+  VERSION = "0.0.1"
+
+  def self.logger
+    return @logger if @logger
+
+    @logger = Logger.new(STDOUT)
+    @logger.level = Logger::WARN
+    @logger
+  end
+
+  def self.config
+    @config ||= Configuration.new
+    yield(@config) if block_given?
+    @config
+  end
+end

data/lib/wayfarer/configuration.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "ostruct"
+require "securerandom"
+require "forwardable"
+
+module Wayfarer
+  class Configuration < OpenStruct
+    extend Forwardable
+
+    DEFAULTS = {
+      # Print full stacktraces?
+      print_stacktraces: true,
+
+      # Crash when encountering unhandled exceptions?
+      reraise_exceptions: false,
+
+      # Allow processing URIs multiple times?
+      allow_circulation: false,
+
+      # How many HTTP connections/Selenium drivers to use
+      # 1:1 correspondence with spawned threads
+      connection_count: 1,
+
+      # Which HTTP adapter to use. Supported are :net_http and :selenium
+      http_adapter: :net_http,
+
+      # Which frontier to use.
+      frontier: :memory,
+
+      # How long a thread may hold an HTTP adapter.
+      # Threads that exceed this limit fail with an exception.
+      connection_timeout: Float::INFINITY,
+
+      # How many 3xx redirects to follow. Has no effect when using Selenium
+      max_http_redirects: 3,
+
+      # Argument vector for instantiating Selenium drivers
+      selenium_argv: [:firefox],
+
+      # Argument vector for instantiating a Redis connection
+      redis_opts: {
+        host: "localhost",
+        port: 6379
+      }.freeze,
+
+      # Size of browser windows
+      window_size: [1024, 768],
+
+      # Which Mustermann pattern type to use when matching URI paths
+      # TODO: Mention in docs
+      mustermann_type: :sinatra,
+
+      # Options for instantiating Bloomfilters
+      bloomfilter_opts: {
+        size: 100,
+        hashes: 2,
+        seed: 1,
+        bucket: 3,
+        raise: false
+      },
+
+      # Whether to normalize URIs
+      normalize_uris: true,
+
+      # URI normalization options
+      # See: https://github.com/rwz/normalize_url
+      normalize_uri_options: {}
+    }.freeze
+
+    attr_reader :uuid
+
+    def initialize(overrides = {})
+      super(DEFAULTS.merge(overrides))
+      @uuid = SecureRandom.uuid
+    end
+
+    def logger
+      @logger ||= Wayfarer.logger.dup
+    end
+
+    def reset!
+      DEFAULTS.each { |key, val| self[key] = val }
+    end
+  end
+end

data/lib/wayfarer/crawl.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "observer"
+require "normalize_url"
+
+module Wayfarer
+  class Crawl
+    extend Forwardable
+    include Observable
+
+    # The prepared job.
+    # @!attribute [r] job
+    attr_reader :job
+
+    # @!attribute [r] dispatcher
+    attr_reader :dispatcher
+
+    delegate config: :job
+    delegate logger: :config
+
+    def initialize(job, *uris)
+      @job = job.prepare
+      @uris = uris
+      @dispatcher = Dispatcher.new(@job)
+      @processor = Processor.new(@job, frontier, @dispatcher)
+    end
+
+    def execute
+      trap_signals
+
+      CrawlObserver.new(@processor, @dispatcher, config.logger)
+
+      @job.run_hook(:before_crawl)
+      @processor.run(*@uris)
+      @job.run_hook(:after_crawl)
+    ensure
+      untrap_signals
+    end
+
+    # A frontier with initially pre-staged URIs.
+    # @return [Frontier]
+    def frontier
+      return @frontier if @frontier
+
+      @frontier = case config.frontier
+                  when :memory_trie
+                    Frontiers::MemoryTrieFrontier.new(config)
+                  when :redis
+                    Frontiers::RedisFrontier.new(config)
+                  when :memory_bloom
+                    Frontiers::MemoryBloomfilter.new(config)
+                  when :redis_bloom
+                    Frontiers::RedisBloomfilter.new(config)
+                  else
+                    Frontiers::MemoryFrontier.new(config)
+                  end
+
+      @frontier.extend(Frontiers::NormalizeURIs) if config.normalize_uris
+
+      @frontier.stage(*@uris) # TODO: Test
+
+      @frontier
+    end
+
+    private
+
+    def trap_signals
+      @cached_sigint_handler = trap(:INT) {
+        halt!
+        @cached_sigint_handler.try(:call)
+        exit(-1)
+      }
+    end
+
+    def untrap_signals
+      trap(:INT) { @cached_sigint_handler.try(:call) }
+    end
+  end
+end

data/lib/wayfarer/crawl_observer.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class CrawlObserver
+    module Events
+      FirstCycle = Struct.new(:frontier)
+      NewCycle = Struct.new(:current_uris_count)
+      DispatchedURI = Struct.new(:action, :uri)
+      CycleFinished = Class.new
+      Peeking = Struct.new(:uri)
+      AboutToCycle = Struct.new(:staged_uris_count)
+      MismatchedURI = Struct.new(:uri)
+      HaltInitiated = Struct.new(:action, :uri)
+      StagingURIs = Struct.new(:staged_uris_count)
+      UnhandledError = Struct.new(:exception)
+    end
+
+    module ObservableShortcuts
+      def notify_observers!(*argv)
+        changed
+        notify_observers(*argv)
+      end
+    end
+
+    extend Forwardable
+
+    attr_reader :logger
+
+    def initialize(*observables, logger)
+      @logger = logger
+      observables.each { |obsv| obsv.add_observer(self) }
+    end
+
+    def update(event)
+      case event
+      when Events::FirstCycle     then first_cycle(event)
+      when Events::NewCycle       then new_cycle(event)
+      when Events::DispatchedURI  then dispatched_uri(event)
+      when Events::CycleFinished  then cycle_finished
+      when Events::Peeking        then peeking(event)
+      when Events::AboutToCycle   then about_to_cycle(event)
+      when Events::MismatchedURI  then mismatched_uri(event)
+      when Events::HaltInitiated  then halt_initiated(event)
+      when Events::StagingURIs    then staging_uris(event)
+      when Events::UnhandledError then unhandled_error(event)
+      end
+    end
+
+    private
+
+    def first_cycle(event)
+      logger.info("First cycle")
+      logger.info("Frontier: #{event.frontier}")
+    end
+
+    def new_cycle(event)
+      logger.info("Current cycle contains #{event.current_uris_count} URI(s)")
+    end
+
+    def dispatched_uri(event)
+      logger.info("Dispatched to \##{event.action}: #{event.uri}")
+    end
+
+    def cycle_finished
+      logger.info("No URIs left in current cycle")
+    end
+
+    def peeking(event)
+      logger.info("Peeking into: #{event.uri}")
+    end
+
+    def about_to_cycle(event)
+      logger.info("About to cycle. #{event.staged_uris_count} staged URI(s)")
+    end
+
+    def mismatched_uri(event)
+      logger.debug("No matching route for: #{event.uri}")
+    end
+
+    def halt_initiated(event)
+      logger.info("Halt initiated from \##{event.action} at: #{event.uri}")
+    end
+
+    def staging_uris(event)
+      logger.info("Staging #{event.staged_uris_count} URI(s)")
+    end
+
+    def unhandled_error(event)
+      level = config.reraise_exceptions ? :fatal : :error
+
+      if config.print_stacktraces
+        logger.public_send level, <<~LOGGER
+          Unhandled exception in an action: #{event.exception.class.inspect}
+          #{event.exception.backtrace.map(&:to_s).join("\n* ")}
+        LOGGER
+      else
+        logger.public_send level, <<~LOGGER
+          Unhandled exception in an action: #{event.exception.class.inspect}
+        LOGGER
+      end
+    end
+  end
+end