wayfarer 0.0.3 → 0.4.0

data/.travis.yml

@@ -1,5 +0,0 @@
-language: ruby
-services: redis-server
-rvm:
-  - 2.3.1
-  - jruby-9.1.6.0

data/Changelog.md

@@ -1,10 +0,0 @@
-## 0.0.2
-
-Features:
-
-* `Job#stage` now expands relative paths/URIs
-* Added `ProtocolRule` and `CustomRule`
-
-Bugfixes:
-
-* Fixed the CLI which relied on a removed method alias

data/README.md

@@ -1,21 +0,0 @@
-# Wayfarer
-[![Build Status](https://travis-ci.org/bauerd/wayfarer.svg)](https://travis-ci.org/bauerd/wayfarer)
-[![Code Climate](https://codeclimate.com/github/bauerd/wayfarer/badges/gpa.svg)](https://codeclimate.com/github/bauerd/wayfarer)
-
-Versatile web crawling with (J)Ruby
-
-* [__Usage and more__ on the website](https://bauerd.github.io/wayfarer/)
-* [__API documentation__ on Ruby-Doc.org](http://www.rubydoc.info/github/bauerd/wayfarer) (`master` branch)
-* __Releases__ on RubyGems.org:
-  * [wayfarer](https://rubygems.org/gems/wayfarer)
-  * [wayfarer-jruby](https://rubygems.org/gems/wayfarer-jruby)
-
-MRI:
-```
-% [sudo] gem install wayfarer
-```
-
-JRuby:
-```
-% [sudo] gem install wayfarer-jruby
-```

data/benchmark/frontiers.rb

@@ -1,143 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../lib/wayfarer"
-
-require "benchmark"
-require "parallel" unless RUBY_PLATFORM == "java"
-require "faker"
-
-include Wayfarer::Frontiers
-
-def print_separator
-  puts "-" * `tput cols`.to_i
-end
-
-# URI count is kept low on purpose because this file is run on Travis CI.
-# Bump up the numbers when running locally.
-URI_COUNT = 10_000
-SAMPLE_COUNT = URI_COUNT * 0.01
-CYCLE_COUNT = 20
-
-FRONTIERS = RUBY_PLATFORM == "java" ? {
-  MemoryFrontier => "memory",
-  RedisFrontier => "redis"
-} : {
-  MemoryBloomfilter => "memory_bloom",
-  MemoryFrontier => "memory",
-  MemoryTrieFrontier => "memory_trie",
-  RedisBloomfilter => "redis_bloom",
-  RedisFrontier => "redis"
-}
-
-label_width = FRONTIERS.reduce(0) { |acc, (_, label)|
-  acc > (l = label.length) ? acc : l
-}
-
-puts "Generating #{URI_COUNT} URIs..."
-
-uris = if RUBY_PLATFORM == "java"
-         Array.new(URI_COUNT) { Faker::Internet.url }
-       else
-         Parallel.map(URI_COUNT.times) { Faker::Internet.url }
-       end
-samples = uris.sample(SAMPLE_COUNT)
-
-puts "Done generating URIs"
-
-print_separator
-puts "Cache insertion"
-
-Benchmark.bm do |bm|
-  FRONTIERS.each do |(klass, label)|
-    frontier = klass.new(Wayfarer::Configuration.new)
-    frontier.cache(*uris)
-
-    bm.report(label.ljust(label_width)) do
-      frontier.cache(*samples)
-    end
-
-    frontier.free
-  end
-end
-
-print_separator
-puts "Cache detection"
-
-Benchmark.bm do |bm|
-  FRONTIERS.each do |(klass, label)|
-    frontier = klass.new(Wayfarer::Configuration.new)
-    frontier.cache(*uris)
-
-    bm.report(label.ljust(label_width)) do
-      samples.each { |sample| frontier.cached?(sample) }
-    end
-
-    frontier.free
-  end
-end
-
-print_separator
-puts "Cycling"
-
-slices = samples.each_slice(CYCLE_COUNT)
-
-Benchmark.bm do |bm|
-  FRONTIERS.each do |(klass, label)|
-    frontier = klass.new(Wayfarer::Configuration.new)
-
-    bm.report(label.ljust(label_width)) do
-      slices.each do |samples|
-        frontier.stage(*samples)
-        frontier.cycle
-      end
-    end
-
-    frontier.free
-  end
-end
-
-print_separator
-puts "Cache detection false positives/negatives"
-
-trials = samples.count / 2
-
-positives = samples.sample(trials)
-negatives = if RUBY_PLATFORM == "java"
-              Array.new(trials) { Faker::Internet.url }
-            else
-              Parallel.map(trials.times) { Faker::Internet.url }
-            end
-
-FRONTIERS.each do |(klass, label)|
-  frontier = klass.new(Wayfarer::Configuration.new)
-
-  false_positives = false_negatives = 0
-
-  positives.each do |positive| frontier.cache(positive) end
-
-  positives.each do |positive|
-    false_positives += 1 unless frontier.cached?(positive)
-  end
-
-  negatives.each do |negative|
-    false_positives += 1 if frontier.cached?(negative)
-  end
-
-  fp_percentage = if false_positives.zero?
-                    0
-                  else
-                    (trials.to_f / false_positives).round(3)
-                  end
-
-  fn_percentage = if false_negatives.zero?
-                    0
-                  else
-                    (trials.to_f / false_negatives).round(3)
-                  end
-
-  puts "* #{label.ljust(label_width)}"
-  puts "  * False positives: #{false_positives} (#{fp_percentage}%)"
-  puts "  * False negatives: #{false_negatives} (#{fn_percentage}%)"
-
-  frontier.free
-end

data/docs/.gitignore

@@ -1,2 +0,0 @@
-_site
-.sass-cache

data/docs/_config.yml

@@ -1,15 +0,0 @@
-gems:
-  - bourbon
-  - neat
-
-markdown: kramdown
-sass:
-  style: :compressed
-
-kramdown:
-  input: GFM
-  syntax_highlighter: rouge
-
-highlighter: rouge
-
-title: Wayfarer

data/docs/_includes/base.html

@@ -1,7 +0,0 @@
-<!-- See: https://ricostacruz.com/til/relative-paths-in-jekyll -->
-{% assign base = '' %}
-{% assign depth = page.url | split: '/' | size | minus: 1 %}
-{% if    depth <= 1 %}{% assign base = '.' %}
-{% elsif depth == 2 %}{% assign base = '..' %}
-{% elsif depth == 3 %}{% assign base = '../..' %}
-{% elsif depth == 4 %}{% assign base = '../../..' %}{% endif %}

data/docs/_includes/head.html

@@ -1,10 +0,0 @@
-<head>
-  <title>{{page.title}} | Wayfarer</title>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width">
-  <link rel="stylesheet" href="{{base}}/css/screen.css">
-  <link
-    rel="stylesheet"
-    href="//brick.a.ssl.fastly.net/Titillium:400,600/Roboto+Mono:400,700">
-  <script src="{{base}}/js/navigation.js"></script>
-</head>

data/docs/_includes/navigation.html

@@ -1,187 +0,0 @@
-<section class="navigation">
-  <input id="navigation__toggle-checkbox" class="navigation__toggle-checkbox" type="checkbox">
-  <label for="navigation__toggle-checkbox" class="navigation__toggle-label">
-    <span class="show">Show navigation</span>
-    <span class="hide">Hide navigation</span>
-  </label>
-  <nav class="navigation__main">
-    <ul class="navigation__list">
-      <li class="navigation__category">
-        <h1 class="navigation__category__title">Guides</h1>
-        <ul class="navigation__list">
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/tutorial.html">
-              Tutorial
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/configuration.html">
-              Configuration
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/halting.html">
-              Halting
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/locals.html">
-              Locals
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/page_objects.html">
-              <code>Page</code> objects
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/error_handling.html">
-              Error handling
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/frontiers.html">
-              (Redis) Frontiers
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/peeking.html">
-              Peeking
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/selenium_capybara.html">
-              Selenium &amp; Capybara
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/callbacks.html">
-              Callbacks
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/cli.html">
-              CLI
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/job_queues.html">
-              Job queues
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/guides/logging.html">
-              Logging
-            </a>
-          </li>
-        </ul>
-      </li>
-
-      <li class="navigation__category">
-        <h1 class="navigation__category__title">Routing</h1>
-        <ul class="navigation__list">
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/routes.html">
-              Routes
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/uri_rules.html">
-              URI rules
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/host_rules.html">
-              Host rules
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/path_rules.html">
-              Path rules
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/query_rules.html">
-              Query rules
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/protocol_rules.html">
-              Protocol rules
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/filetypes_rules.html">
-              Filetypes rules
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/routing/custom_rules.html">
-              Custom rules
-            </a>
-          </li>
-        </ul>
-      </li>
-
-      <li class="navigation__category">
-        <h1 class="navigation__category__title">Recipes</h1>
-        <ul class="navigation__list">
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/recipes/multiple_uris.html">
-              Starting from multiple URIs
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/recipes/authentication.html">
-              Authentication
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/recipes/screenshots.html">
-              Taking screenshots
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/recipes/javascript.html">
-              Executing JavaScript
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/recipes/csv.html">
-              CSV output
-            </a>
-          </li>
-        </ul>
-      </li>
-
-      <li class="navigation__category">
-        <h1 class="navigation__category__title">Miscellaneous</h1>
-        <ul class="navigation__list">
-          <li class="navigation__page">
-            <a class="navigation__link" href="//github.com/bauerd/wayfarer">
-              Code on GitHub
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="http://www.rubydoc.info/github/bauerd/wayfarer">
-              API documentation
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/misc/testing.html">
-              Testing
-            </a>
-          </li>
-          <li class="navigation__page">
-            <a class="navigation__link" href="{{base}}/misc/contributing.html">
-              Contributing
-            </a>
-          </li>
-        </ul>
-      </li>
-
-    <label for="navigation__toggle-checkbox" class="navigation__close-bottom">
-      Hide navigation
-    </label>
-  </nav>
-</section>

data/docs/_layouts/default.html

@@ -1,42 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  {% include base.html %}
-  {% include head.html %}
-
-  <body>
-    <div class="wrapper">
-      <aside class="sidebar">
-        <header class="site-header">
-          <h1 class="site-header__title">
-            <a href="{{base}}" class="site-header__link">{{ site.title }}</a>
-          </h1>
-          <div class="site-header__version">
-            0.0.3
-          </div>
-        </header>
-
-        <section class="badges">
-          <iframe class="star-button" src="https://ghbtns.com/github-btn.html?user=bauerd&repo=wayfarer&type=star" frameborder="0" scrolling="0" width="50px" height="20px"></iframe>
-          <img class="build-status" src="https://travis-ci.org/bauerd/wayfarer.svg?branch=master">
-        </section>
-
-        {% include navigation.html %}
-      </aside>
-
-      <main class="page-content">
-        {{content}}
-
-        <aside class="page-meta">
-          Instructions unclear or not working? Please
-          <a href="{{site.github.repository_url}}/edit/master/docs/{{page.path}}">
-            edit this page
-          </a>
-          or
-          <a href="{{site.github.repository_url}}/issues/new">
-            file a bug
-          </a>
-        </aside>
-      </main>
-    </div>
-  </body>
-</html>