wayfarer 0.4.5 → 0.4.7

data/lib/wayfarer/redis/resettable.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Redis
+    module Resettable
+      def reset!
+        redis_pool.with { |conn| conn.del(redis_key) }
+      end
+
+      def redis_pool
+        raise NotImplementedError
+      end
+
+      def redis_key
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/dsl.rb

@@ -13,6 +13,7 @@ module Wayfarer
 
       def path(path, options = {}, &block)
         offset = File.join(path_offset, path)
+
         add_child_route(nil, offset, options, &block).tap do |route|
           route.matcher = Matchers::Path.new(offset, route)
         end

data/lib/wayfarer/routing/matchers/path.rb

@@ -17,10 +17,12 @@ module Wayfarer
         end
 
         def match(url)
-          route.accept(self)
-
+          # TODO: Ditch this and add a parameter to the path DSL#path
+          #
           # If the route's branch contains other path matchers in child routes,
           # match the beginning of the path (peeking), instead of the whole path.
+          route.accept(self)
+
           !!(if peeking
                matcher.peek(url.path)
              else

data/lib/wayfarer/routing/root_route.rb

@@ -2,6 +2,10 @@
 
 module Wayfarer
   module Routing
-    class RootRoute < Route; end
+    class RootRoute < Route
+      def invoke(url)
+        PathFinder.result(self, url)
+      end
+    end
   end
 end

data/lib/wayfarer/routing/route.rb

@@ -5,6 +5,7 @@ module Wayfarer
     class Route
       include DSL
       include Stringify
+      extend Forwardable
 
       attr_reader :children
 
@@ -15,8 +16,7 @@ module Wayfarer
 
       stringify :matcher,
                 :action,
-                :path_offset,
-                :children
+                :path_offset
 
       def initialize(matcher = Matchers::Custom.new { children.any? }, path_offset = "/")
         @matcher = matcher
@@ -24,24 +24,14 @@ module Wayfarer
         @path_offset = path_offset
       end
 
-      def match(url)
-        matcher.match(url)
-      end
-
-      def matches?(url)
-        invoke(url).is_a?(Result::Match)
-      end
-
-      def invoke(url)
-        PathFinder.result(self, url)
-      end
-
       # Accepts a visitor for in-order traversal.
       def accept(visitor)
         return unless visitor.visit(self)
 
         children.each { |child| child.accept(visitor) }
       end
+
+      delegate match: :matcher
     end
   end
 end

data/lib/wayfarer/stringify.rb

@@ -1,46 +1,38 @@
 # frozen_string_literal: true
 
-# 346325
-
 module Wayfarer
   module Stringify
-    def self.included(base)
-      base.include(InstanceMethods)
-      base.extend(ClassMethods)
-      base.instance_eval do
-        cattr_accessor :stringified_attributes do
-          []
-        end
-      end
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :stringified_attributes,
+                      default: [],
+                      instance_accessor: false,
+                      instance_predicate: false
+
+      alias_method :inspect, :to_s
     end
 
-    module ClassMethods
+    class_methods do
       def stringify(*variables)
         stringified_attributes.concat(variables)
       end
     end
 
-    module InstanceMethods
-      def to_s
-        if self.class.stringified_attributes.any?
-          "#<#{class_name} #{attributes.join(', ')}>"
-        else
-          "#<#{class_name}>"
-        end
-      end
-
-      alias inspect to_s
+    def to_s
+      class_name = self.class.name
 
-      def class_name
-        self.class.name
-      end
+      if self.class.stringified_attributes.any?
+        attrs = self.class
+                    .stringified_attributes
+                    .map { |attr| [attr, public_send(attr)] }
+                    .to_h
+                    .map { |k, v| "#{k}=#{v.inspect}" }
+                    .join(", ")
 
-      def attributes
-        self.class
-            .stringified_attributes
-            .map { |attr| [attr, public_send(attr)] }
-            .to_h
-            .map { |k, v| [k, "=", v.inspect].join }
+        "#<#{class_name} #{attrs}>"
+      else
+        "#<#{class_name}>"
       end
     end
   end

data/lib/wayfarer/task.rb

@@ -1,36 +1,30 @@
 # frozen_string_literal: true
 
 module Wayfarer
+  # @!attribute [r] url
+  #   @return [String] the URL to process
+  # @!attribute [r] batch
+  #   @return [String] the batch the task belongs to
   class Task
+    extend Forwardable
     include Stringify
 
-    attr_reader :url,
-                :batch,
-                :metadata
+    attr_reader :url, :batch
 
-    stringify :url,
-              :batch
+    stringify :url, :batch
 
+    delegate %i([] []=) => :@ephemeral
+
+    # @!visibility private
     def initialize(url, batch)
       @url = url
       @batch = batch
-      @metadata = OpenStruct.new
+      @ephemeral = {}
     end
 
+    # @!visibility private
     def ==(other)
       [url, batch] == [other.url, other.batch]
     end
-
-    def barrier
-      @barrier ||= Wayfarer::Redis::Barrier.new(batch)
-    end
-
-    def counter
-      @counter ||= Wayfarer::Redis::Counter.new(batch)
-    end
-
-    def gc
-      @gc ||= Wayfarer::GC.new(self)
-    end
   end
 end

data/lib/wayfarer.rb

@@ -38,11 +38,36 @@ module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 5
+    TINY = 6
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
-  mattr_accessor :config, default: Wayfarer::Config::Root.new
+  DEFAULT_CONFIG = {
+    redis: {
+      url: "redis://localhost:6379/0",
+      factory: ->(redis) { ::Redis.new(url: redis[:url]) }
+    },
+    network: {
+      agent: :http,
+      pool_size: 1,
+      pool_timeout: 10,
+      http_headers: {},
+      renew_on: []
+    },
+    capybara: {
+      driver: nil
+    },
+    ferrum: {
+      options: {}
+    },
+    selenium: {
+      driver: :chrome,
+      options: {},
+      client_timeout: 60
+    }
+  }.freeze
+
+  mattr_accessor :config, default: DEFAULT_CONFIG.clone
 
   UserAgentTimeoutError = Class.new(StandardError) # TODO: Move to Networking namespace
 end
@@ -50,3 +75,5 @@ end
 loader.eager_load
 
 ActiveJob::Serializers.serializers << Wayfarer::Serializer
+
+Wayfarer::BatchCompletion.subscribe!

data/mkdocs.yml

@@ -1,14 +1,20 @@
 repo_url: https://github.com/bauerd/wayfarer
 edit_uri: edit/develop/docs/
-repo_name: Source Code
+repo_name: Code
 site_name: Wayfarer
 markdown_extensions:
   - admonition
+  - attr_list
   - meta
   - def_list
   - pymdownx.details
   - pymdownx.highlight
-  - pymdownx.superfences
+  - pymdownx.inlinehilite
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
   - pymdownx.critic
   - pymdownx.caret
   - pymdownx.mark
@@ -18,11 +24,15 @@ markdown_extensions:
       custom_checkbox: true
 
 theme:
+  icon:
+    logo: material/sign-direction
+    repo: fontawesome/brands/git
   name: material
   features:
     - navigation.tabs
-    - navigation.sections
-    - navigation.expand
+    # - navigation.sections
+    # - navigation.expand
+    - navigation.indexes
   font:
     text: IBM Plex Sans
     code: IBM Plex Sans Mono
@@ -41,7 +51,42 @@ theme:
       toggle:
         icon: material/lightbulb
         name: Switch to light mode
-  icon:
-    repo: fontawesome/brands/git
-
 
+nav:
+  - Home: index.md
+  - Guides:
+    - Tutorial: guides/tutorial.md
+    - Jobs:
+      - Overview: guides/jobs.md
+      - Error handling: guides/jobs/error_handling.md
+    - Tasks: guides/tasks.md
+    - Pages: guides/pages.md
+    - Routing:
+        - Overview: guides/routing.md
+        - Matchers:
+          - URL: todo
+          - Host: todo
+          - Path: todo
+          - Query: todo
+          - Custom matchers: todo
+    - Callbacks: guides/callbacks.md
+    - Handlers: guides/handlers.md
+    - Networking:
+        - Introduction: guides/user_agents.md
+        - User agent API: guides/networking/custom_adapters.md
+        - Built-in user agents:
+          - Plain HTTP: guides/networking/http.md
+          - Ferrum: guides/networking/ferrum.md
+          - Selenium: guides/networking/selenium.md
+          - Capybara: guides/networking/capybara.md
+    - Redis: guides/redis.md
+    - Design decisions: design.md
+  - Reference:
+    - Configuration: reference/configuration.md
+    - Command-line interface: reference/cli.md
+  - Cookbook:
+    - Browser navigation: cookbook/navigation.md
+    - Executing JavaScript: cookbook/executing_javascript.md
+    - Screenhots: cookbook/screenshots.md
+    - Setting the User-Agent: cookbook/user_agent.md
+  - API documentation: "https://www.rubydoc.info"

data/rake/docs.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "yard"
+
+namespace :yard do
+  desc "Generate documentation"
+  YARD::Rake::YardocTask.new(:generate) do |t|
+    require_relative "../lib/wayfarer"
+
+    t.options = %w[--readme docs/index.md --no-private --markup markdown]
+    t.files = [
+      Wayfarer::Base,
+      Wayfarer::Task,
+      Wayfarer::Page,
+      Wayfarer::Handler,
+      Wayfarer::Routing::DSL,
+      Wayfarer::Parsing
+    ].freeze.map { |klass| Object.const_source_location(klass.name).first }
+  end
+
+  desc "Regenerate documentation on change"
+  task :watch do
+    # The output of `yard server` differs for some reason
+    sh "rerun --wait 0.5 --dir lib bundle exec rake yard:generate"
+  end
+end

data/rake/lint.rake

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "open-uri"
+require "fileutils"
+require "rubygems/package"
+require "zlib"
+require "zip"
+require "rubocop/rake_task"
+require "net/http"
+require "uri"
+
+VALE_URL_ROOT = "https://github.com/errata-ai"
+VALE_VERSION = "2.15.4"
+VALE_STYLE_GUIDE_VERSION = "v0.3.3"
+VALE_PREFIX = File.expand_path(File.join(".github", "bin"))
+VALE_STYLE_GUIDE_PREFIX = File.expand_path(File.join(".github", "share"))
+
+namespace :lint do
+  RuboCop::RakeTask.new do |task|
+    task.formatters = %w[simple]
+  end
+
+  vale_command = File.join(VALE_PREFIX, "vale")
+  style_guide_directory = File.join(VALE_STYLE_GUIDE_PREFIX, "vale")
+
+  desc "Lint documentation"
+  task vale: [:"lint:vale:clean", vale_command, style_guide_directory] do
+    sh "#{vale_command} docs/**/*.md"
+  end
+
+  namespace :vale do
+    directory VALE_PREFIX
+
+    desc "Install Vale to #{vale_command}"
+    file vale_command => VALE_PREFIX do
+      unless (local_command = `which vale`.strip).empty?
+        next File.symlink(local_command, vale_command)
+      end
+
+      filename = "vale_#{VALE_VERSION}_Linux_64-bit.tar.gz"
+      url = File.join(VALE_URL_ROOT, "/vale/releases/download/v#{VALE_VERSION}/#{filename}")
+
+      extract_tar_gz(url, vale_command)
+
+      FileUtils.chmod("+x", vale_command)
+    end
+
+    desc "Deletes Vale"
+    task clean: :"lint:vale:style_guide:clean" do
+      File.delete(vale_command) if File.exist?(vale_command)
+    end
+
+    namespace :style_guide do
+      directory VALE_STYLE_GUIDE_PREFIX
+
+      desc "Retrieve Vale Google style guide #{VALE_STYLE_GUIDE_VERSION}"
+      directory style_guide_directory => VALE_STYLE_GUIDE_PREFIX do
+        FileUtils.mkdir_p(style_guide_directory)
+        url = "https://github.com/errata-ai/Google/releases/download/#{VALE_STYLE_GUIDE_VERSION}/Google.zip"
+        extract_zip(url, style_guide_directory)
+      end
+
+      desc "Deletes the Vale Google style guide"
+      task :clean do
+        FileUtils.rm_rf([style_guide_directory])
+      end
+
+    private
+
+      def extract_zip(url, destination)
+        content = URI.open(url)
+
+        Zip::File.open_buffer(content) do |zip|
+          zip.each do |entry|
+            path = File.join(destination, entry.name)
+
+            if entry.directory?
+              FileUtils.mkdir_p(path)
+            else
+              entry.extract(path)
+              File.chmod(0o755, path)
+            end
+          end
+        end
+      end
+    end
+
+  private
+
+    def extract_tar_gz(url, destination)
+      URI.open(url) do |file|
+        Zlib::GzipReader.open(file) do |gz|
+          Gem::Package::TarReader.new(gz) do |tar|
+            tar.each do |entry|
+              next unless entry.file?
+
+              binary = entry.read
+              File.write(destination, binary)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/rake/release.rake

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "yaml"
+require "bundler/gem_tasks"
+
+namespace :release do
+  task :guard_versions do
+    gem_spec = Gem::Specification.load("wayfarer.gemspec")
+    gem_version = gem_spec.version.version
+    lib_version = Wayfarer::VERSION::STRING
+
+    raise "Gem version #{gem_version} deviates from library version #{lib_version}" unless gem_version == lib_version
+  end
+
+  # TODO: `GEM_HOST_API_KEY` is only supported for RubyGems 3+
+  task :write_credentials do
+    raise "RubyGems 3+ supports `GEM_HOST_API_KEY`" unless RUBY_VERSION.split(".").first.to_i == 2
+
+    key = ENV.fetch("GEM_HOST_API_KEY") { raise "`GEM_HOST_API_KEY` is unset" }
+    contents = YAML.dump(rubygems_api_key: "Basic #{key}")
+    gem_path = File.join(Dir.home, ".gem")
+    FileUtils.mkdir_p(gem_path)
+    File.write(File.join(gem_path, "credentials"), contents)
+  end
+end
+
+Rake::Task[:release].enhance(%i[release:guard_versions])
+Rake::Task[:"release:rubygem_push"].enhance(%i[release:write_credentials])

data/rake/tests.rake

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "rspec/core/rake_task"
+
+desc "Run all tests"
+RSpec::Core::RakeTask.new(:test)
+
+namespace :test do
+  desc "Run only isolated tests"
+  RSpec::Core::RakeTask.new :isolated do |task|
+    task.rspec_opts = ["--tag ~selenium --tag ~ferrum --tag ~cli"]
+  end
+
+  desc "Run only Selenium tests"
+  RSpec::Core::RakeTask.new :selenium do |task|
+    task.rspec_opts = ["--tag selenium"]
+  end
+
+  desc "Run only Ferrum tests"
+  RSpec::Core::RakeTask.new :ferrum do |task|
+    task.rspec_opts = ["--tag ferrum"]
+  end
+
+  desc "Run only CLI tests"
+  RSpec::Core::RakeTask.new :cli do |task|
+    task.rspec_opts = ["--tag cli"]
+  end
+end

data/requirements.txt

@@ -1 +1 @@
-mkdocs-material == 7.3.0
+mkdocs-material == 9.5.9