wayfarer 0.0.3 → 0.4.3

data/docs/misc/testing.md

@@ -1,11 +0,0 @@
----
-layout: default
-title: Testing
----
-
-# Testing
-
-Tests run on MRI 2.3.1 and JRuby 9.1.6.0.
-
-* Run `rake -T` to list all available (test) tasks.
-* When running tests, a HTTP server binds to port 9876 for integration-level tests.

data/docs/recipes/authentication.md

@@ -1,23 +0,0 @@
----
-layout: default
-title: Authentication
----
-
-# Authentication
-
-Authentication is best handled with the `setup_adapter` callback See [Callbacks]({{base}}/guides/callbacks.html).
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config.http_adapter = :selenium
-
-  setup_adapter do |_, _, browser|
-    browser.visit("https://foo.com/login")
-
-    browser.fill_in("E-mail", with: "foo@bar.com")
-    browser.fill_in("Password", with: "password")
-
-    browser.click_button("Log in")
-  end
-end
-{% endhighlight %}

data/docs/recipes/csv.md

@@ -1,29 +0,0 @@
----
-layout: default
-title: CSV output
----
-
-# CSV output
-
-See:
-
-* [Locals]({{base}}/guides/locals.html)
-* [Callbacks]({{base}}/guides/callbacks.html)
-
-{% highlight ruby %}
-require "csv" # from Ruby's standard lib
-
-class DummyJob < Wayfarer::Job
-  let(:records) { [] }
-
-  after_crawl do
-    CSV.open("output.csv", "w") do |csv|
-      records.each { |r| csv << [r[:id], r[:name]] }
-    end
-  end
-
-  def detail
-    records << { id: ..., name: ...  }
-  end
-end
-{% endhighlight %}

data/docs/recipes/javascript.md

@@ -1,20 +0,0 @@
----
-layout: default
-title: Executing JavaScript
----
-
-# Executing JavaScript
-
-In order to execute JavaScript in a page's DOM context, use the Selenium HTTP adapter and call `#execute_script` on the WebDriver object:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config.http_adapter = :selenium
-
-  # ...
-
-  def foo
-    pathname = driver.execute_script("return window.location.pathname")
-  end
-end
-{% endhighlight %}

data/docs/recipes/multiple_uris.md

@@ -1,18 +0,0 @@
----
-layout: default
-title: Starting from multiple URIs
----
-
-# Starting from multiple URIs
-
-You can pass in as many URIs as desired when performing jobs:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  # ...
-end
-
-uris = [...]
-
-DummyJob.perform_now(*uris)
-{% endhighlight %}

data/docs/recipes/screenshots.md

@@ -1,20 +0,0 @@
----
-layout: default
-title: Taking screenshots
----
-
-# Taking screenshots
-
-In order to take screenshots, use the Selenium HTTP adapter and call `#save_screenshot` on the WebDriver object:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config.http_adapter = :selenium
-
-  # ...
-
-  def foo
-    driver.save_screenshot("my_screenshot.png")
-  end
-end
-{% endhighlight %}

data/docs/routing/custom_rules.md

@@ -1,16 +0,0 @@
----
-layout: default
-title: Custom rules
----
-
-# Custom rules
-
-Custom rules take a block that gets yielded the URI or an object that responds to `#call(uri)`. If the block or the delegate return a truthy value, the rule matches.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.if -> (uri) { uri.host == uri.host.reverse }
-end
-{% endhighlight %}
-
-* Matches only URIs with palindrome hosts

data/docs/routing/filetypes_rules.md

@@ -1,21 +0,0 @@
----
-layout: default
-title: Filetypes rules
----
-
-# Filetypes rules
-
-Filetypes rules match against the URI path's file extension.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.filetypes [:png, :jpg], to: :image
-  route.forbid.filetypes [:php, :js]
-end
-{% endhighlight %}
-
-Matches:
-
-* `http://example.com/foo.png`
-* `http://example.com/foo.jpg`
-* `https://example.com/qux/bar.jpg`

data/docs/routing/host_rules.md

@@ -1,24 +0,0 @@
----
-layout: default
-title: Host rules
----
-
-# Host rules
-
-Host rules match against a host string or RegExp.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.host "example.com"
-  route.host /example/
-end
-{% endhighlight %}
-
-Matches:
-
-* All URIs hosted on `"example.com"`.
-* All URIs that contain `"example"`.
-
-<aside class="note">
-<code>"www.host.net"</code> and <code>"host.net"</code> are not considered equal. You have to specify the exact host when using strings. Consider using <code>/host.net/</code> instead.
-</aside>

data/docs/routing/path_rules.md

@@ -1,33 +0,0 @@
----
-layout: default
-title: Path rules
----
-
-# Path rules
-
-Path rules match against the path of a URI. Both strings and RegExps are accepted, and path segment pattern matching and RegExp captures are supported.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.path "/:alpha/:beta", to: :foo
-  route.path /^foobar\/(.+)/, to: :bar
-
-  def foo
-    params[:alpha]
-    params[:beta]
-  end
-
-  def foo
-    params["0"]
-  end
-end
-{% endhighlight %}
-
-Matches:
-
-* All URIs with path segments matching `/:alpha/:beta`, e.g. `https://example.com/foo/bar`
-* All URIs starting with `"/foobar/"`.
-
-<aside class="note">
-<code>/:alpha/:beta</code> and <code>:alpha/:beta</code> are not considered equal. Note the opening slash.
-</aside>

data/docs/routing/protocol_rules.md

@@ -1,17 +0,0 @@
----
-layout: default
-title: Protocol rules
----
-
-# Protocol rules
-
-Protocol rules match against symbols/strings.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.protocol :https
-end
-{% endhighlight %}
-
-* Matches `https://example.com`.
-* Does not match `http://example.com`.

data/docs/routing/query_rules.md

@@ -1,69 +0,0 @@
----
-layout: default
-title: Query rules
----
-
-# Query rules
-
-Query rules impose constraints on key-value query parameters. Strings, integers, RegExps and ranges are supported.
-
-## String constraints
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.query arg: "foo"
-end
-{% endhighlight %}
-
-* Matches `https://example.com?arg=foo`.
-
-## Integer constraints
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.query arg: 42
-end
-{% endhighlight %}
-
-* Matches `https://example.com?arg=42`.
-
----
-
-## RegExp constraints
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.query arg: /foo/
-end
-{% endhighlight %}
-
-* Matches `https://example.com?arg=foo`.
-* Matches `https://example.com?arg=foobar`.
-
----
-
-## Range constraints
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.query arg: 1..10
-end
-{% endhighlight %}
-
-* Matches `https://example.com?arg=1`.
-* Matches […]
-* Matches `https://example.com?arg=10`.
-
----
-
-## Compound constraints
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.query foo: 1..5, bar: /baz/, qux: "zot", toto: 2
-end
-{% endhighlight %}
-
-* Matches `https://example.com?foo=4&bar=bazqux&qux=zot&toto=2`.
-
----

data/docs/routing/routes.md

@@ -1,96 +0,0 @@
----
-layout: default
-title: Routes
-categories: [Routing]
----
-
-# Routes
-
-* Routes are filters for interesting URIs.
-* Routes put constraints on URIs that should get processed.
-* Routes map URIs to instance methods (actions).
-* Routes are tree nodes and thus nestable.
-
-Currently, the following rules are available:
-
-* [URI rules](uri_rules.html) match URIs against a string.
-* [Host rules](/routing/host_rules.html) match hosts against strings and RegExps.
-* [Path rules](/routing/path_rules.html) match paths against pattern strings and RegExps. They support  path segment capturing.
-* [Query rules](/routing/query_rules.html) match key-value pairs of query parameters against strings, integers, RegExps and ranges.
-
-Routes can be fordidden. URIs that match forbidden rules are never processed.
-
-## Route declaration
-
-### Declaration order matching
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.host "example.com", to: :foo
-  route.path "/foo", to: :bar
-
-  # Is equivalent to:
-  #
-  # routes do
-  #   host "example.com", to: :foo
-  #   path "/foo", to: :bar
-  # end
-end
-{% endhighlight %}
-
-* Dispatches `https://example.com/foo` to `:foo`.
-* Dispatches `https://example.com` to `:foo`.
-* Dispatches `https://yahoo.com/foo` to `:bar`.
-
----
-
-### Nesting routes (child rules)
-
-A route matches if it has a child rule that matches. This applies recursively.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.host "example.com", to: :foo do
-    path "/foo"
-  end
-
-  # Is equivalent to:
-  #
-  # route.host "example.com", path: "/foo", to: :foo
-  # route.path "/foo", host: "example.com", to: :foo
-end
-{% endhighlight %}
-
-* Dispatches `https://example.com/foo` to `:foo`.
-* Does not dispatch `https://example.com`.
-* Does not dispatch `https://yahoo.com/foo`.
-
----
-
-### Deepest routes override actions
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.host "example.com", to: :foo do
-    path "/foo", to: :bar
-  end
-end
-{% endhighlight %}
-
-* Dispatches `https://example.com/foo` to `:bar`.
-* Does not dispatch `https://example.com`.
-* Does not dispatch `https://yahoo.com/foo`.
-
----
-
-### Forbidding rules
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.forbid.path "/foo"
-  route.host "example.com", to: :foo
-end
-{% endhighlight %}
-
-* Dispatches `https://example.com` to `:bar`.
-* Does not dispatch `https://example.com/foo`.

data/docs/routing/uri_rules.md

@@ -1,18 +0,0 @@
----
-layout: default
-title: URI Rules
----
-
-# URI rules
-
-URI rules match against a string.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  route.uri "https://example.com"
-end
-{% endhighlight %}
-
-Matches:
-
-* Only `https://example.com`

data/examples/collect_github_issues.rb

@@ -1,65 +0,0 @@
-require_relative "../lib/wayfarer"
-
-class CollectGithubIssues < Wayfarer::Job
-  config.connection_count = 4
-  config.logger.level = :fatal
-
-  let(:records) { [] }
-
-  routes do
-    host "github.com" do
-      path "/:user/:repo",            to: :repository
-      path "/:user/:repo/issues",     to: :index
-      path "/:user/:repo/issues/:id", to: :show
-    end
-  end
-
-  after_crawl do
-    records.each do |issue|
-      # Save them somewhere?
-      puts issue
-    end
-  end
-
-  def repository
-    stage navigation_links
-  end
-
-  def index
-    stage issue_listing_links, next_page
-  end
-
-  def show
-    return halt if records.count > 30
-
-    records << {
-      id: params[:id],
-      title: issue_title,
-      author: issue_author
-    }
-  end
-
-  private
-
-  def issue_title
-    doc.css(".js-issue-title").text.strip
-  end
-
-  def issue_author
-    doc.css(".TableObject-item .author").text.strip
-  end
-
-  def navigation_links
-    page.links ".reponav-item"
-  end
-
-  def issue_listing_links
-    page.links ".issues-listing"
-  end
-
-  def next_page
-    page.links ".next_page"
-  end
-end
-
-CollectGithubIssues.perform_now("https://github.com/rails/rails")

data/examples/find_foobar_on_wikipedia.rb

@@ -1,23 +0,0 @@
-require_relative "../lib/wayfarer"
-
-class FindFoobarOnWikipedia < Wayfarer::Job
-  config.http_adapter = :selenium
-  config.selenium_argv = [:chrome]
-  config.connection_count = 4
-
-  let(:keywords) { [] }
-
-  route.host "en.wikipedia.org", to: :article
-
-  def article
-    if page.body =~ /Foobar/
-      driver.save_screenshot("/tmp/foobar.png")
-      return halt
-    end
-
-    keywords << page.keywords
-    stage page.links
-  end
-end
-
-FindFoobarOnWikipedia.perform_now("https://en.wikipedia.org/wiki/Special:Random")

data/lib/wayfarer/configuration.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-require "ostruct"
-require "securerandom"
-require "forwardable"
-
-module Wayfarer
-  class Configuration < OpenStruct
-    extend Forwardable
-
-    DEFAULTS = {
-      # Print full stacktraces?
-      print_stacktraces: true,
-
-      # Crash when encountering unhandled exceptions?
-      reraise_exceptions: false,
-
-      # Allow processing URIs multiple times?
-      allow_circulation: false,
-
-      # How many HTTP connections/Selenium drivers to use
-      # 1:1 correspondence with spawned threads
-      connection_count: 1,
-
-      # Which HTTP adapter to use. Supported are :net_http and :selenium
-      http_adapter: :net_http,
-
-      # Which frontier to use.
-      frontier: :memory,
-
-      # How long a thread may hold an HTTP adapter.
-      # Threads that exceed this limit fail with an exception.
-      connection_timeout: Float::INFINITY,
-
-      # How many 3xx redirects to follow. Has no effect when using Selenium
-      max_http_redirects: 3,
-
-      # Argument vector for instantiating Selenium drivers
-      selenium_argv: [:firefox],
-
-      # Argument vector for instantiating a Redis connection
-      redis_opts: {
-        host: "localhost",
-        port: 6379
-      }.freeze,
-
-      # Size of browser windows
-      window_size: [1024, 768],
-
-      # Which Mustermann pattern type to use when matching URI paths
-      # TODO: Mention in docs
-      mustermann_type: :sinatra,
-
-      # Options for instantiating Bloomfilters
-      bloomfilter_opts: {
-        size: 100,
-        hashes: 2,
-        seed: 1,
-        bucket: 3,
-        raise: false
-      },
-
-      # Whether to normalize URIs
-      normalize_uris: true,
-
-      # URI normalization options
-      # See: https://github.com/rwz/normalize_url
-      normalize_uri_options: {}
-    }.freeze
-
-    attr_reader :uuid
-
-    def initialize(overrides = {})
-      super(DEFAULTS.merge(overrides))
-      @uuid = SecureRandom.uuid
-    end
-
-    def logger
-      @logger ||= Wayfarer.logger.dup
-    end
-
-    def reset!
-      DEFAULTS.each { |key, val| self[key] = val }
-    end
-  end
-end

data/lib/wayfarer/crawl.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-require "observer"
-require "normalize_url"
-
-module Wayfarer
-  class Crawl
-    extend Forwardable
-    include Observable
-
-    # The prepared job.
-    # @!attribute [r] job
-    attr_reader :job
-
-    # @!attribute [r] dispatcher
-    attr_reader :dispatcher
-
-    delegate config: :job
-    delegate logger: :config
-
-    def initialize(job, *uris)
-      @job = job.prepare
-      @uris = uris
-      @dispatcher = Dispatcher.new(@job)
-      @processor = Processor.new(@job, frontier, @dispatcher)
-    end
-
-    def execute
-      trap_signals
-
-      CrawlObserver.new(@processor, @dispatcher, config.logger)
-
-      @job.run_hook(:before_crawl)
-      @processor.run(*@uris)
-      @job.run_hook(:after_crawl)
-    ensure
-      untrap_signals
-    end
-
-    # A frontier with initially pre-staged URIs.
-    # @return [Frontier]
-    def frontier
-      return @frontier if @frontier
-
-      @frontier = case config.frontier
-                  when :memory_trie
-                    Frontiers::MemoryTrieFrontier.new(config)
-                  when :redis
-                    Frontiers::RedisFrontier.new(config)
-                  when :memory_bloom
-                    Frontiers::MemoryBloomfilter.new(config)
-                  when :redis_bloom
-                    Frontiers::RedisBloomfilter.new(config)
-                  else
-                    Frontiers::MemoryFrontier.new(config)
-                  end
-
-      @frontier.extend(Frontiers::NormalizeURIs) if config.normalize_uris
-
-      @frontier.stage(*@uris) # TODO: Test
-
-      @frontier
-    end
-
-    private
-
-    def trap_signals
-      @cached_sigint_handler = trap(:INT) {
-        halt!
-        @cached_sigint_handler.try(:call)
-        exit(-1)
-      }
-    end
-
-    def untrap_signals
-      trap(:INT) { @cached_sigint_handler.try(:call) }
-    end
-  end
-end