wayfarer 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c99ff804578236db96766eecce89450d6e489f4d
+  data.tar.gz: 8340d4c0e55cb35c20fc688f6ac9badfb6b4a6cc
+SHA512:
+  metadata.gz: 10308569f8290771a7fce4e5b382c19103cd29671d539fc85917852e6194722ee9b8c8d49f08bc8034bf63f604b13a369fdfffb69ed67703ec3c2ca25d490352
+  data.tar.gz: 2726271b256bab8781267dd6851ba0fda6601fea2fae38f3ed7adf68c0ba64190d3da0fcb38cf4bc0de39a7c3ddfe3507c32714a2fbcc5fbf813659dbe0459bc

data/.gitignore

@@ -0,0 +1,8 @@
+.DS_Store
+.bundle
+.yardoc
+Gemfile.lock
+*.gem
+build/
+bench/
+yard/

data/.rbenv-gemsets

@@ -0,0 +1 @@
+wayfarer

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--tty

data/.rubocop.yml

@@ -0,0 +1,21 @@
+AllCops:
+  Exclude:
+    - "spec/**/*"
+    - "examples/**/*"
+    - "support/**/*"
+    - "wayfarer.gemspec"
+    - "examples"
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/BlockDelimiters:
+  EnforcedStyle: semantic
+  FunctionalMethods:
+    - expect
+
+Metrics/MethodLength:
+  Max: 25
+
+Metrics/PerceivedComplexity:
+  Max: 10

data/.ruby-version

@@ -0,0 +1 @@
+2.3.1

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+services: redis-server
+rvm:
+  - 2.3.1
+  - jruby-9.1.6.0

data/.yardopts

@@ -0,0 +1,3 @@
+--output-dir yard
+--markup markdown
+--protected

data/Changelog.md

@@ -0,0 +1,10 @@
+## 0.0.2
+
+Features:
+
+* `Job#stage` now expands relative paths/URIs
+* Added `ProtocolRule` and `CustomRule`
+
+Bugfixes:
+
+* Fixed the CLI which relied on a removed method alias

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "github-pages", group: :jekyll_plugins
+
+gemspec name: if RUBY_PLATFORM == "java"
+                "wayfarer-jruby"
+              else
+                "wayfarer"
+              end

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 Dominic Bauer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,21 @@
+# Wayfarer
+[![Build Status](https://travis-ci.org/bauerd/wayfarer.svg)](https://travis-ci.org/bauerd/wayfarer)
+[![Code Climate](https://codeclimate.com/github/bauerd/wayfarer/badges/gpa.svg)](https://codeclimate.com/github/bauerd/wayfarer)
+
+Versatile web crawling with (J)Ruby
+
+* [__Usage and more__ on the website](https://bauerd.github.io/wayfarer/)
+* [__API documentation__ on Ruby-Doc.org](http://www.rubydoc.info/github/bauerd/wayfarer) (`master` branch)
+* __Releases__ on RubyGems.org:
+  * [wayfarer](https://rubygems.org/gems/wayfarer)
+  * [wayfarer-jruby](https://rubygems.org/gems/wayfarer-jruby)
+
+MRI:
+```
+% [sudo] gem install wayfarer
+```
+
+JRuby:
+```
+% [sudo] gem install wayfarer-jruby
+```

data/Rakefile

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "rspec/core/rake_task"
+require "rubocop/rake_task"
+require "rack"
+
+require_relative "support/test_app"
+
+task default: ["ci"]
+
+namespace :test do
+  desc "Run only environment-agnostic tests"
+  RSpec::Core::RakeTask.new isolated: [:test_app] do |task|
+    task.rspec_opts = ["--tag ~selenium --tag ~redis"]
+  end
+
+  desc "Run only integration tests"
+  RSpec::Core::RakeTask.new integration: [:test_app] do |task|
+    task.rspec_opts = ["--tag integration"]
+  end
+
+  desc "Run only Selenium tests"
+  RSpec::Core::RakeTask.new selenium: [:test_app] do |task|
+    task.rspec_opts = ["--tag selenium"]
+  end
+
+  desc "Run only Redis tests"
+  RSpec::Core::RakeTask.new redis: [:test_app] do |task|
+    task.rspec_opts = ["--tag redis"]
+  end
+end
+
+desc "Run all tests"
+RSpec::Core::RakeTask.new(test: :test_app)
+
+RuboCop::RakeTask.new do |task|
+  task.formatters = ["simple"]
+end
+
+desc "Run benchmarks"
+task :benchmark do
+  bms = Rake::FileList["benchmark/*.rb"]
+
+  bms.each do |bm|
+    sh "bundle exec ruby #{bm}"
+  end
+end
+
+desc "CI run"
+task ci: %i[test benchmark]
+
+namespace :build do
+  desc "Build the MRI gem"
+  task :mri do
+    sh "gem build wayfarer.gemspec --verbose"
+    sh "mkdir -p build"
+    sh "mv wayfarer-*.gem build"
+  end
+
+  desc "Build the MRI gem"
+  task :jruby do
+    sh "gem build wayfarer-jruby.gemspec --verbose"
+    sh "mkdir -p build"
+    sh "mv wayfarer-jruby-*.gem build"
+  end
+end
+
+desc "Remove build directory"
+task :clean do
+  sh "rm -rf build"
+end
+
+desc "Start a preloaded REPL"
+task :repl do
+  require_relative "lib/wayfarer"
+
+  include Wayfarer
+  include Wayfarer::HTTPAdapters
+  include Wayfarer::Routing
+  include Wayfarer::Frontiers
+
+  begin
+    require "pry"
+    Pry.new.repl(self)
+  rescue LoadError
+    require "irb"
+    ARGV.clear
+    IRB.start
+  end
+end
+
+desc %(List lines that contain "FIXME" or "TODO")
+task :todo do
+  sh %(grep -rn "\\(FIXME\\|TODO\\)" lib spec | tr -s [:space:])
+end
+
+task :test_app do
+  mutex = Mutex.new
+  cvar  = ConditionVariable.new
+
+  Thread.new do
+    Rack::Handler::WEBrick.run(
+      TestApp,
+      Port: 9876,
+      BindAddress: "localhost",
+      Logger: WEBrick::Log.new("/dev/null"),
+      AccessLog: [],
+      StartCallback: proc { cvar.signal }
+    )
+  end
+
+  mutex.lock
+  cvar.wait(mutex)
+end

data/benchmark/frontiers.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require_relative "../lib/wayfarer"
+
+require "benchmark"
+require "parallel" unless RUBY_PLATFORM == "java"
+require "faker"
+
+include Wayfarer::Frontiers
+
+def print_separator
+  puts "-" * `tput cols`.to_i
+end
+
+# URI count is kept low on purpose because this file is run on Travis CI.
+# Bump up the numbers when running locally.
+URI_COUNT = 10_000
+SAMPLE_COUNT = URI_COUNT * 0.01
+CYCLE_COUNT = 20
+
+FRONTIERS = RUBY_PLATFORM == "java" ? {
+  MemoryFrontier => "memory",
+  RedisFrontier => "redis"
+} : {
+  MemoryBloomfilter => "memory_bloom",
+  MemoryFrontier => "memory",
+  MemoryTrieFrontier => "memory_trie",
+  RedisBloomfilter => "redis_bloom",
+  RedisFrontier => "redis"
+}
+
+label_width = FRONTIERS.reduce(0) { |acc, (_, label)|
+  acc > (l = label.length) ? acc : l
+}
+
+puts "Generating #{URI_COUNT} URIs..."
+
+uris = if RUBY_PLATFORM == "java"
+         Array.new(URI_COUNT) { Faker::Internet.url }
+       else
+         Parallel.map(URI_COUNT.times) { Faker::Internet.url }
+       end
+samples = uris.sample(SAMPLE_COUNT)
+
+puts "Done generating URIs"
+
+print_separator
+puts "Cache insertion"
+
+Benchmark.bm do |bm|
+  FRONTIERS.each do |(klass, label)|
+    frontier = klass.new(Wayfarer::Configuration.new)
+    frontier.cache(*uris)
+
+    bm.report(label.ljust(label_width)) do
+      frontier.cache(*samples)
+    end
+
+    frontier.free
+  end
+end
+
+print_separator
+puts "Cache detection"
+
+Benchmark.bm do |bm|
+  FRONTIERS.each do |(klass, label)|
+    frontier = klass.new(Wayfarer::Configuration.new)
+    frontier.cache(*uris)
+
+    bm.report(label.ljust(label_width)) do
+      samples.each { |sample| frontier.cached?(sample) }
+    end
+
+    frontier.free
+  end
+end
+
+print_separator
+puts "Cycling"
+
+slices = samples.each_slice(CYCLE_COUNT)
+
+Benchmark.bm do |bm|
+  FRONTIERS.each do |(klass, label)|
+    frontier = klass.new(Wayfarer::Configuration.new)
+
+    bm.report(label.ljust(label_width)) do
+      slices.each do |samples|
+        frontier.stage(*samples)
+        frontier.cycle
+      end
+    end
+
+    frontier.free
+  end
+end
+
+print_separator
+puts "Cache detection false positives/negatives"
+
+trials = samples.count / 2
+
+positives = samples.sample(trials)
+negatives = if RUBY_PLATFORM == "java"
+              Array.new(trials) { Faker::Internet.url }
+            else
+              Parallel.map(trials.times) { Faker::Internet.url }
+            end
+
+FRONTIERS.each do |(klass, label)|
+  frontier = klass.new(Wayfarer::Configuration.new)
+
+  false_positives = false_negatives = 0
+
+  positives.each do |positive| frontier.cache(positive) end
+
+  positives.each do |positive|
+    false_positives += 1 unless frontier.cached?(positive)
+  end
+
+  negatives.each do |negative|
+    false_positives += 1 if frontier.cached?(negative)
+  end
+
+  fp_percentage = if false_positives.zero?
+                    0
+                  else
+                    (trials.to_f / false_positives).round(3)
+                  end
+
+  fn_percentage = if false_negatives.zero?
+                    0
+                  else
+                    (trials.to_f / false_negatives).round(3)
+                  end
+
+  puts "* #{label.ljust(label_width)}"
+  puts "  * False positives: #{false_positives} (#{fp_percentage}%)"
+  puts "  * False negatives: #{false_negatives} (#{fn_percentage}%)"
+
+  frontier.free
+end

data/bin/wayfarer

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "logger"
+require "pathname"
+require "time"
+require "active_support/inflector"
+require "thor"
+
+require_relative "../lib/wayfarer"
+
+module Wayfarer
+  class CLI < Thor
+    class_option :log_level, type: :string, default: "info"
+
+    desc "exec FILE URI",
+         "Run the job defined in FILE starting from URI(s)"
+    def exec(file, uri)
+      set_log_level(options)
+      load_class_from_file(file).perform(uri)
+    end
+
+    option :queue_adapter, type: :string, default: "inline"
+    option :wait, type: :string
+    desc "enqueue FILE URI",
+         "Run the job defined in FILE starting from URI(s)"
+    def enqueue(file, uri)
+      set_log_level(options)
+      set_queue_adapter(options)
+
+      if wait = options[:wait]
+        delayed_seconds = differing_seconds_from_string(wait)
+        load_class_from_file(file).set(wait: delayed_seconds).perform_later(uri)
+      else
+        load_class_from_file(file).perform_later(uri)
+      end
+    end
+
+    desc "route FILE URI",
+         "Output the first matching route from FILE for URI"
+    def route(file, uri)
+      set_log_level(options)
+
+      job = load_class_from_file(file)
+
+      action, params = job.router.route(URI(uri))
+
+      if action
+        puts "Dispatching to :#{action} with parameters: #{params}"
+      else
+        puts "No matching route"
+      end
+    end
+
+    desc "version", "Print the version"
+    def version
+      puts Wayfarer::VERSION
+    end
+
+    private
+
+    def set_log_level(options)
+      if level = options[:log_level]
+        Wayfarer.logger.level = log_level_from_string(level)
+      end
+    end
+
+    def set_queue_adapter(options)
+      if adapter = options[:queue_adapter]
+        ActiveJob::Base.queue_adapter = adapter
+      end
+    end
+
+    def log_level_from_string(str)
+      case str.downcase
+      when "unknown" then Logger::UNKNOWN
+      when "debug"   then Logger::DEBUG
+      when "error"   then Logger::ERROR
+      when "fatal"   then Logger::FATAL
+      when "info"    then Logger::INFO
+      when "warn"    then Logger::WARN
+      end
+    end
+
+    def differing_seconds_from_string(str)
+      # RuboCop turned this into an art piece
+      if secs = begin
+                  Integer(secs)
+                rescue
+                  nil
+                end
+        secs
+      elsif time = begin
+                     Time.parse(str)
+                   rescue
+                     nil
+                   end
+        time - Time.now
+      else
+        # TODO: Get rid of this
+        require "chronic"
+        Chronic.parse(str) - Time.now
+      end
+    end
+
+    def load_class_from_file(file)
+      path = File.join(Dir.pwd, file.chomp(".rb"))
+      base_name = Pathname.new(path).basename.to_s
+
+      require path
+      base_name.camelize.constantize
+    end
+  end
+end
+
+Wayfarer::CLI.start(ARGV)

data/docs/.gitignore

@@ -0,0 +1,2 @@
+_site
+.sass-cache

data/docs/_config.yml

@@ -0,0 +1,15 @@
+gems:
+  - bourbon
+  - neat
+
+markdown: kramdown
+sass:
+  style: :compressed
+
+kramdown:
+  input: GFM
+  syntax_highlighter: rouge
+
+highlighter: rouge
+
+title: Wayfarer

data/docs/_includes/base.html

@@ -0,0 +1,7 @@
+<!-- See: https://ricostacruz.com/til/relative-paths-in-jekyll -->
+{% assign base = '' %}
+{% assign depth = page.url | split: '/' | size | minus: 1 %}
+{% if    depth <= 1 %}{% assign base = '.' %}
+{% elsif depth == 2 %}{% assign base = '..' %}
+{% elsif depth == 3 %}{% assign base = '../..' %}
+{% elsif depth == 4 %}{% assign base = '../../..' %}{% endif %}

data/docs/_includes/head.html

@@ -0,0 +1,10 @@
+<head>
+  <title>{{page.title}} | Wayfarer</title>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width">
+  <link rel="stylesheet" href="{{base}}/css/screen.css">
+  <link
+    rel="stylesheet"
+    href="//brick.a.ssl.fastly.net/Titillium:400,600/Roboto+Mono:400,700">
+  <script src="{{base}}/js/navigation.js"></script>
+</head>