wayfarer 0.0.3 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c99ff804578236db96766eecce89450d6e489f4d
-  data.tar.gz: 8340d4c0e55cb35c20fc688f6ac9badfb6b4a6cc
+SHA256:
+  metadata.gz: b6fd1a3fe8b9b259d023e8ffb125819c8663a4c5a3ead1930b46c673198276bd
+  data.tar.gz: e98395a58d7ab9e1062319f7ffe308056d8bfeae857463f8031e463e83aed5d8
 SHA512:
-  metadata.gz: 10308569f8290771a7fce4e5b382c19103cd29671d539fc85917852e6194722ee9b8c8d49f08bc8034bf63f604b13a369fdfffb69ed67703ec3c2ca25d490352
-  data.tar.gz: 2726271b256bab8781267dd6851ba0fda6601fea2fae38f3ed7adf68c0ba64190d3da0fcb38cf4bc0de39a7c3ddfe3507c32714a2fbcc5fbf813659dbe0459bc
+  metadata.gz: d1d08a347ffc7cd54721522d7babc13ac91d50a2c06caa6bbceb13d6d633e6636e62316abd387f5c06250ed8d58802b6b766dbe2da1e8510bf638c64d05833ce
+  data.tar.gz: 185f44131bf7cebbb20eded4b83b4bf1bace6c3c41cc807e35bcda51191f6f18cba63b8dd63f963c3e24643c8eb772370cf4a6a36b00d19628853833324c4bae

data/.github/workflows/ci.yaml

@@ -0,0 +1,32 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - '*'
+env:
+  CI: true
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Start services
+        run: docker-compose up -d
+
+      - name: Run isolated tests
+        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:isolated
+
+      - name: Run Ferrum tests
+        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:ferrum
+
+      - name: Run Selenium tests
+        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:selenium
+
+      - name: Run CLI tests
+        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake test:cli
+
+      - name: Run RuboCop
+        run: docker-compose run --rm --name test --service-ports wayfarer bundle exec rake rubocop

data/.gitignore

@@ -1,8 +1,7 @@
 .DS_Store
 .bundle
 .yardoc
-Gemfile.lock
 *.gem
-build/
-bench/
-yard/
+build
+yard
+\#*#

data/.rubocop.yml

@@ -1,21 +1,37 @@
 AllCops:
+  NewCops: enable
   Exclude:
-    - "spec/**/*"
-    - "examples/**/*"
-    - "support/**/*"
-    - "wayfarer.gemspec"
-    - "examples"
+    - "*.gemspec"
+
+Layout/AccessModifierIndentation:
+  EnforcedStyle: outdent
+
+Layout/LineLength:
+  Max: 120
 
 Style/StringLiterals:
   EnforcedStyle: double_quotes
 
 Style/BlockDelimiters:
-  EnforcedStyle: semantic
+  EnforcedStyle: braces_for_chaining
   FunctionalMethods:
+    - let
+    - let!
+    - subject
     - expect
 
+Style/Documentation:
+  Enabled: false
+
 Metrics/MethodLength:
-  Max: 25
+  Max: 20
+
+Metrics/BlockLength:
+  Exclude:
+    - "spec/**/*"
+
+Metrics/AbcSize:
+  Max: 30
 
-Metrics/PerceivedComplexity:
-  Max: 10
+Metrics/CyclomaticComplexity:
+  Max: 15

data/.ruby-version

@@ -1 +1 @@
-2.3.1
+2.7.4

data/Dockerfile

@@ -0,0 +1,5 @@
+FROM ruby:2.7.4
+ENV CI=true
+WORKDIR /usr/src/app
+COPY . .
+RUN bundle install

data/Gemfile

@@ -2,10 +2,4 @@
 
 source "https://rubygems.org"
 
-gem "github-pages", group: :jekyll_plugins
-
-gemspec name: if RUBY_PLATFORM == "java"
-                "wayfarer-jruby"
-              else
-                "wayfarer"
-              end
+gemspec name: "wayfarer"

data/Gemfile.lock

@@ -0,0 +1,221 @@
+PATH
+  remote: .
+  specs:
+    wayfarer (0.4.3)
+      activejob (>= 6.0)
+      addressable (~> 2.8)
+      capybara (~> 3.0)
+      connection_pool (~> 2.2)
+      docile (~> 1.1)
+      ferrum (~> 0.9)
+      metainspector (~> 5.0)
+      mime-types (~> 3.0)
+      mock_redis (~> 0.29)
+      mustermann (~> 1.1)
+      net-http-persistent (~> 3.0)
+      nokogiri (~> 1.11)
+      normalize_url (~> 0.0.6)
+      redis (~> 4.4, < 4.5)
+      selenium-webdriver (~> 3.4)
+      thor (~> 1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activejob (6.1.4.1)
+      activesupport (= 6.1.4.1)
+      globalid (>= 0.3.6)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    capybara (3.36.0)
+      addressable
+      matrix
+      mini_mime (>= 0.1.3)
+      nokogiri (~> 1.8)
+      rack (>= 1.6.0)
+      rack-test (>= 0.6.3)
+      regexp_parser (>= 1.5, < 3.0)
+      xpath (~> 3.2)
+    childprocess (3.0.0)
+    cliver (0.3.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    cuprite (0.13)
+      capybara (>= 2.1, < 4)
+      ferrum (~> 0.11.0)
+    diff-lcs (1.4.4)
+    docile (1.4.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    factory_bot (6.2.0)
+      activesupport (>= 5.0.0)
+    faker (1.9.6)
+      i18n (>= 0.7)
+    faraday (1.9.3)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.0)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      faraday-retry (~> 1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
+      http-cookie (~> 1.0.0)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-encoding (0.0.5)
+      faraday
+    faraday-excon (1.1.0)
+    faraday-http-cache (2.2.0)
+      faraday (>= 0.8)
+    faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
+      multipart-post (>= 1.2, < 3)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    faraday-retry (1.0.3)
+    faraday_middleware (1.2.0)
+      faraday (~> 1.0)
+    fastimage (2.2.6)
+    ferrum (0.11)
+      addressable (~> 2.5)
+      cliver (~> 0.3)
+      concurrent-ruby (~> 1.1)
+      websocket-driver (>= 0.6, < 0.8)
+    globalid (1.0.0)
+      activesupport (>= 5.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    matrix (0.4.2)
+    metainspector (5.11.2)
+      addressable (~> 2.7)
+      faraday (~> 1.4)
+      faraday-cookie_jar (~> 0.0)
+      faraday-encoding (~> 0.0)
+      faraday-http-cache (~> 2.2)
+      faraday_middleware (~> 1.0)
+      fastimage (~> 2.2)
+      nesty (~> 1.0)
+      nokogiri (~> 1.11)
+    method_source (1.0.0)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2022.0105)
+    mini_mime (1.1.2)
+    mini_portile2 (2.6.1)
+    minitest (5.14.4)
+    mock_redis (0.29.0)
+      ruby2_keywords
+    multipart-post (2.1.1)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    nesty (1.0.2)
+    net-http-persistent (3.1.0)
+      connection_pool (~> 2.2)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
+      racc (~> 1.4)
+    normalize_url (0.0.6)
+      addressable (~> 2.3)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.6)
+    racc (1.6.0)
+    rack (1.6.13)
+    rack-protection (1.5.5)
+      rack
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    redis (4.4.0)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (0.93.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.5)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 0.6.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.11.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    rubyzip (2.3.2)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
+    sinatra (1.4.8)
+      rack (~> 1.5)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    thor (1.2.1)
+    tilt (2.0.10)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8)
+    unicode-display_width (1.8.0)
+    websocket-driver (0.7.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    xpath (3.2.0)
+      nokogiri (~> 1.8)
+    yard (0.9.26)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  cuprite (~> 0.13)
+  factory_bot (~> 6.0)
+  faker (~> 1.7)
+  pry (~> 0.10)
+  rake (~> 10.4)
+  rspec (~> 3.4)
+  rubocop (~> 0.49)
+  sinatra (~> 1.4)
+  wayfarer!
+  yard (~> 0.9)
+
+BUNDLED WITH
+   2.1.4

data/RELEASING.md

@@ -0,0 +1,17 @@
+# Release Procedure
+
+1. Ensure `Wayfarer::VERSION` was bumped appropriately.
+2. Ensure the version in wayfarer.gemspec matches.
+3. Open a release Pull Request develop -> master branch
+4. Merge the Pull Request
+5. Publish RubyGem and git tag as follows:
+
+```
+git checkout master
+git pull origin master --rebase
+bundle exec rake build
+gem push build/wayfarer-*.gem
+bundle exec rake clean
+git tag <VERSION>
+git push origin <VERSION>
+```

data/Rakefile

@@ -1,114 +1,62 @@
 # frozen_string_literal: true
 
+require "irb"
 require "rspec/core/rake_task"
 require "rubocop/rake_task"
-require "rack"
 
-require_relative "support/test_app"
+BUILD_DIR = "build"
 
-task default: ["ci"]
+task default: :build
 
-namespace :test do
-  desc "Run only environment-agnostic tests"
-  RSpec::Core::RakeTask.new isolated: [:test_app] do |task|
-    task.rspec_opts = ["--tag ~selenium --tag ~redis"]
-  end
-
-  desc "Run only integration tests"
-  RSpec::Core::RakeTask.new integration: [:test_app] do |task|
-    task.rspec_opts = ["--tag integration"]
-  end
-
-  desc "Run only Selenium tests"
-  RSpec::Core::RakeTask.new selenium: [:test_app] do |task|
-    task.rspec_opts = ["--tag selenium"]
-  end
-
-  desc "Run only Redis tests"
-  RSpec::Core::RakeTask.new redis: [:test_app] do |task|
-    task.rspec_opts = ["--tag redis"]
-  end
+desc "Start an IRB session"
+task :console do
+  require_relative "lib/wayfarer"
+  ARGV.clear
+  IRB.start
 end
 
-desc "Run all tests"
-RSpec::Core::RakeTask.new(test: :test_app)
-
-RuboCop::RakeTask.new do |task|
-  task.formatters = ["simple"]
+desc "Build gem"
+task build: %i[clean] do
+  sh "gem build wayfarer.gemspec --verbose"
+  sh "mkdir #{BUILD_DIR}"
+  sh "mv wayfarer-*.gem #{BUILD_DIR}"
 end
 
-desc "Run benchmarks"
-task :benchmark do
-  bms = Rake::FileList["benchmark/*.rb"]
-
-  bms.each do |bm|
-    sh "bundle exec ruby #{bm}"
-  end
+desc "Remove build directory"
+task :clean do
+  sh "rm -rf #{BUILD_DIR}"
 end
 
-desc "CI run"
-task ci: %i[test benchmark]
+desc "Install gem"
+task install: %i[build] do
+  sh "gem install #{BUILD_DIR}/*.gem"
+end
 
-namespace :build do
-  desc "Build the MRI gem"
-  task :mri do
-    sh "gem build wayfarer.gemspec --verbose"
-    sh "mkdir -p build"
-    sh "mv wayfarer-*.gem build"
+namespace :test do
+  desc "Run only isolated tests"
+  RSpec::Core::RakeTask.new :isolated do |task|
+    task.rspec_opts = ["--tag ~selenium --tag ~ferrum --tag ~cli"]
   end
 
-  desc "Build the MRI gem"
-  task :jruby do
-    sh "gem build wayfarer-jruby.gemspec --verbose"
-    sh "mkdir -p build"
-    sh "mv wayfarer-jruby-*.gem build"
+  desc "Run only Selenium tests"
+  RSpec::Core::RakeTask.new :selenium do |task|
+    task.rspec_opts = ["--tag selenium"]
   end
-end
-
-desc "Remove build directory"
-task :clean do
-  sh "rm -rf build"
-end
 
-desc "Start a preloaded REPL"
-task :repl do
-  require_relative "lib/wayfarer"
-
-  include Wayfarer
-  include Wayfarer::HTTPAdapters
-  include Wayfarer::Routing
-  include Wayfarer::Frontiers
-
-  begin
-    require "pry"
-    Pry.new.repl(self)
-  rescue LoadError
-    require "irb"
-    ARGV.clear
-    IRB.start
+  desc "Run only Ferrum tests"
+  RSpec::Core::RakeTask.new :ferrum do |task|
+    task.rspec_opts = ["--tag ferrum"]
   end
-end
 
-desc %(List lines that contain "FIXME" or "TODO")
-task :todo do
-  sh %(grep -rn "\\(FIXME\\|TODO\\)" lib spec | tr -s [:space:])
+  desc "Run only CLI tests"
+  RSpec::Core::RakeTask.new :cli do |task|
+    task.rspec_opts = ["--tag cli"]
+  end
 end
 
-task :test_app do
-  mutex = Mutex.new
-  cvar  = ConditionVariable.new
-
-  Thread.new do
-    Rack::Handler::WEBrick.run(
-      TestApp,
-      Port: 9876,
-      BindAddress: "localhost",
-      Logger: WEBrick::Log.new("/dev/null"),
-      AccessLog: [],
-      StartCallback: proc { cvar.signal }
-    )
-  end
+desc "Run all tests"
+RSpec::Core::RakeTask.new(:test)
 
-  mutex.lock
-  cvar.wait(mutex)
+RuboCop::RakeTask.new do |task|
+  task.formatters = %w[simple]
 end

data/bin/wayfarer

@@ -1,116 +1,6 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "logger"
-require "pathname"
-require "time"
-require "active_support/inflector"
-require "thor"
-
 require_relative "../lib/wayfarer"
 
-module Wayfarer
-  class CLI < Thor
-    class_option :log_level, type: :string, default: "info"
-
-    desc "exec FILE URI",
-         "Run the job defined in FILE starting from URI(s)"
-    def exec(file, uri)
-      set_log_level(options)
-      load_class_from_file(file).perform(uri)
-    end
-
-    option :queue_adapter, type: :string, default: "inline"
-    option :wait, type: :string
-    desc "enqueue FILE URI",
-         "Run the job defined in FILE starting from URI(s)"
-    def enqueue(file, uri)
-      set_log_level(options)
-      set_queue_adapter(options)
-
-      if wait = options[:wait]
-        delayed_seconds = differing_seconds_from_string(wait)
-        load_class_from_file(file).set(wait: delayed_seconds).perform_later(uri)
-      else
-        load_class_from_file(file).perform_later(uri)
-      end
-    end
-
-    desc "route FILE URI",
-         "Output the first matching route from FILE for URI"
-    def route(file, uri)
-      set_log_level(options)
-
-      job = load_class_from_file(file)
-
-      action, params = job.router.route(URI(uri))
-
-      if action
-        puts "Dispatching to :#{action} with parameters: #{params}"
-      else
-        puts "No matching route"
-      end
-    end
-
-    desc "version", "Print the version"
-    def version
-      puts Wayfarer::VERSION
-    end
-
-    private
-
-    def set_log_level(options)
-      if level = options[:log_level]
-        Wayfarer.logger.level = log_level_from_string(level)
-      end
-    end
-
-    def set_queue_adapter(options)
-      if adapter = options[:queue_adapter]
-        ActiveJob::Base.queue_adapter = adapter
-      end
-    end
-
-    def log_level_from_string(str)
-      case str.downcase
-      when "unknown" then Logger::UNKNOWN
-      when "debug"   then Logger::DEBUG
-      when "error"   then Logger::ERROR
-      when "fatal"   then Logger::FATAL
-      when "info"    then Logger::INFO
-      when "warn"    then Logger::WARN
-      end
-    end
-
-    def differing_seconds_from_string(str)
-      # RuboCop turned this into an art piece
-      if secs = begin
-                  Integer(secs)
-                rescue
-                  nil
-                end
-        secs
-      elsif time = begin
-                     Time.parse(str)
-                   rescue
-                     nil
-                   end
-        time - Time.now
-      else
-        # TODO: Get rid of this
-        require "chronic"
-        Chronic.parse(str) - Time.now
-      end
-    end
-
-    def load_class_from_file(file)
-      path = File.join(Dir.pwd, file.chomp(".rb"))
-      base_name = Pathname.new(path).basename.to_s
-
-      require path
-      base_name.camelize.constantize
-    end
-  end
-end
-
-Wayfarer::CLI.start(ARGV)
+Wayfarer::CLI::Runner.start(ARGV)

data/docker-compose.yml

@@ -0,0 +1,32 @@
+version: "3"
+services:
+  wayfarer:
+    build: .
+    volumes:
+    - "./:/usr/src/app"
+    ports:
+    - "9876:9876"
+    environment:
+    - CI=true
+
+  redis:
+    image: redis
+
+  chrome:
+    image: browserless/chrome
+    ports:
+    - "3000:3000"
+
+  firefox:
+    image: selenium/standalone-firefox:4.0.0-rc-2-prerelease-20210923
+    ports:
+    - "4444:4444"
+    volumes:
+    - "/dev/shm:/dev/shm"
+
+  docs:
+    image: squidfunk/mkdocs-material:7.3.0
+    volumes:
+    - "./:/docs"
+    ports:
+    - "8000:8000"

data/docs/cookbook/batch_routing.md

@@ -0,0 +1,22 @@
+# Batch routing
+
+```ruby
+# Create a record in an external database and store the hostname
+record = Database::Row.create(hostname: "example.com")
+
+class DummyJob < Wayfarer::Base
+  route do |hostname|
+    host hostname, to: :index
+  end
+
+  steer do |task|
+    # Pass the external record's hostname to the router
+    [Database::Row.find(task.batch).hostname]
+  end
+
+  # ...
+end
+
+# Enqueue the task and use the database record's key as batch
+DummyJob.crawl_later("https://example.com", batch: record.id)
+```