wayfarer 0.0.3 → 0.4.3

data/docs/reference/environment_variables.md

@@ -0,0 +1,83 @@
+# Environment Variables
+
+## String formats
+
+Environment variable values can be parsed to Hash or Array at runtime
+with the following syntaxes:
+
+* Hash: Variable string `a:1,b:2,c:3` parses to `{a:1, b:2, c:3}` at runtime
+* Array: Variable string `a,b,c` parses to `[:a, :b, :c]` at runtime
+
+## Variables
+
+### `WAYFARER_AGENT`
+:   Either `ferrum`, `selenium` or `http`.
+
+    * Type: String
+    * Key: `config.agent`
+    * Default value: `:http`
+
+### `WAYFARER_POOL_SIZE`
+:   Number of user agents to maintain.
+
+    * Type: Integer
+    * Key: `config.pool_size`
+    * Default value: `1`
+
+### `WAYFARER_POOL_TIMEOUT`
+:   How long a user agent may remain checked out until the owning job
+    fails.
+
+    * Type: Integer
+    * Key: `config.agent_pool_timeout`
+    * Default value: `1`
+
+---
+
+### `WAYFARER_FERRUM_OPTIONS`
+:   Key/value options passed to `Ferrum::Browser.new`.
+
+    * Type: Hash
+    * Key: `config.ferrum_options`
+    * Default value: `{}`
+
+---
+
+### `WAYFARER_SELENIUM_DRIVER`
+:   Driver passed to `Selenium::WebDriver.for`.
+
+    * Type: Symbol
+    * Key: `config.selenium_driver`
+    * Default value: `:chrome`
+
+---
+
+### `WAYFARER_SELENIUM_OPTIONS`
+:   Options passed to `Selenium::WebDriver.for`.
+
+    * Type: Hash
+    * Key: `config.selenium_options`
+    * Default value: `{}`
+    
+---
+
+### `WAYFARER_SELENIUM_CLIENT_TIMEOUT`
+:   Selenium HTTP client timeout (seconds).
+
+    * Type: Integer
+    * Key: `config.selenium_client_timeout`
+    * Default value: `60`
+
+---
+
+### `WAYFARER_HTTP_HEADERS`
+:   HTTP request headers used when retrieving pages.
+
+    * Type: Hash
+    * Key: `config.http_headers`
+    * Default value: `{}`
+    
+    !!! attention "Partial support"
+
+        Selenium does not support configuring HTTP request headers.
+

data/lib/wayfarer/base.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Base < ActiveJob::Base
+    include Wayfarer::Middleware::Worker
+    extend Forwardable
+
+    after_enqueue do |_job|
+      task.counter.increment
+    end
+
+    after_perform { |job| GC.new(job).run }
+
+    rescue_from(StandardError) { GC.new(self).run }
+
+    def self.retry_on(*argv)
+      super(*argv) do |job, error|
+        job.task.barrier.seen?(job.task.url)
+        GC.new(job).run
+        yield job, error if block_given?
+      end
+    end
+
+    def self.discard_on(*argv)
+      super(*argv) do |job, error|
+        job.task.barrier.seen?(job.task.url)
+        GC.new(job).run
+        yield job, error if block_given?
+      end
+    end
+
+    def self.crawl_later(url, batch: SecureRandom.uuid)
+      perform_later(Task.new(url, batch))
+    end
+
+    def retry_job(...)
+      super(...) # increments the counter by re-enqueuing the job
+      task.counter.decrement
+    end
+
+    def perform(task)
+      task.job = self
+      chain.call(task)
+    end
+
+    def task
+      arguments.first
+    end
+  end
+end

data/lib/wayfarer/callbacks.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Callbacks
+    TERMINATOR = ->(_target, result) { result.call == false }
+    OPTIONS = { terminator: TERMINATOR, skip_after_callbacks_if_terminated: true }.freeze
+
+    ConditionalCallback = Struct.new(:job, :filters) do
+      def run(method, &block)
+        return if only && !applies?(only)
+        return if except && applies?(except)
+
+        return job.send(method) if method
+
+        job.instance_eval(&block)
+      end
+
+    private
+
+      def applies?(condition)
+        case condition
+        when Symbol then condition == action
+        when Enumerable then condition&.include?(action)
+        end
+      end
+
+      def only
+        filters[:only]
+      end
+
+      def except
+        filters[:except]
+      end
+
+      def action
+        task.metadata.action
+      end
+
+      def task
+        job.task
+      end
+    end
+
+    def self.included(base)
+      base.include(ActiveSupport::Callbacks)
+      base.extend(ClassMethods)
+
+      base.class_eval do
+        define_callbacks(:fetch, OPTIONS)
+        define_callbacks(:action, OPTIONS)
+        define_callbacks(:batch, OPTIONS)
+
+        define(:fetch, :before)
+        define(:action, :before)
+        define(:batch, :after)
+      end
+    end
+
+    module ClassMethods
+    private
+
+      def define(name, stage)
+        define_singleton_method([stage, name].join("_")) do |method = nil, **filters, &block|
+          set_callback(name, stage, **filters) do |job|
+            ConditionalCallback.new(job, filters).run(method, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/base.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Base < Thor
+      include Thor::Actions
+
+      def self.source_root
+        File.expand_path("templates", __dir__)
+      end
+
+    private
+
+      def mock_redis
+        Wayfarer.config.redis.factory = ->(_) { MockRedis.new }
+      end
+
+      def load_environment
+        Wayfarer::CLI::Runner.loader.setup
+      end
+
+      def free_agent_pool
+        Wayfarer::Networking::Pool.instance.free
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/generate.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Generate < Base
+      desc "project NAME",
+           "Generate new project directory NAME"
+      def project(name)
+        @name = name
+
+        empty_directory name
+        template "Gemfile.tt", "#{name}/Gemfile"
+        template "job.rb.tt", "#{name}/app/jobs/#{name}.rb"
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/job.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Job < Base
+      desc "perform JOB URL",
+           "Perform JOB with URL"
+      option :mock_redis, type: :boolean
+      option :batch, type: :string, default: SecureRandom.uuid
+      def perform(job, url)
+        load_environment
+        mock_redis if options[:mock_redis]
+
+        url = Addressable::URI.parse(url)
+        job = job.classify.constantize.new
+        task = Wayfarer::Task.new(url, options[:batch])
+        job.arguments.push(task)
+        job.perform(task)
+        GC.new(job).run
+      end
+
+      desc "enqueue JOB URL",
+           "Enqueue JOB with URL"
+      option :batch, type: :string, default: SecureRandom.uuid
+      def enqueue(job, url)
+        load_environment
+        mock_redis if options[:mock_redis]
+
+        url = Addressable::URI.parse(url)
+        job = job.classify.constantize
+        job.crawl_later(url, batch: options[:batch])
+      end
+
+      desc "execute JOB URL",
+           "Execute JOB with async adapter"
+      option :mock_redis, type: :boolean
+      option :batch, type: :string, default: SecureRandom.uuid
+      option :min_threads, type: :numeric, default: 1
+      option :max_threads, type: :numeric, default: 1
+      def execute(job, url)
+        load_environment
+        mock_redis if options[:mock_redis]
+
+        url = Addressable::URI.parse(url)
+        job = job.classify.constantize
+
+        job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
+                                                                       max_threads: options[:max_threads])
+        scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
+        executor = scheduler.instance_variable_get(:@async_executor)
+
+        job.crawl_later(url, batch: options[:batch])
+
+        sleep(1) while executor.scheduled_task_count > executor.completed_task_count
+
+        free_agent_pool
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/route.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# # frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Route < Base
+      desc "result JOB URL",
+           "Invoke JOB's router with URL"
+      def result(job, url)
+        load_environment
+        url = Addressable::URI.parse(url)
+        job = job.classify.constantize
+        job.router.invoke(url, job.new.steer)
+        say Wayfarer::Routing::PathFinder.result(job.router.root, url)
+      end
+
+      desc "tree JOB URL",
+           "Visualize JOB's routing tree for URL"
+      def tree(job, url)
+        load_environment
+        url = Addressable::URI.parse(url)
+        job = job.classify.constantize
+        job.router.invoke(url, job.new.steer)
+        Wayfarer::CLI::RoutePrinter.print(job.router.root, url)
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/route_printer.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class RoutePrinter < Thor::Shell::Color
+      attr_reader :url,
+                  :path_finder,
+                  :output
+
+      INDENT           = "   "
+      REGULAR_SEGMENT  = "│  "
+      JUNCTION_SEGMENT = "├──"
+      CORNER_SEGMENT   = "└──"
+
+      def self.print(route, url)
+        route.accept(new(url))
+      end
+
+      def initialize(url)
+        @url = url
+        @path_finder = Wayfarer::Routing::PathFinder.new(url)
+        super()
+      end
+
+      def visit(route)
+        route.accept(path_finder) unless route.parent
+        return true if route.is_a?(Wayfarer::Routing::RootRoute)
+
+        puts [segments(route), label(route)].join("")[3..]
+        true
+      end
+
+      def segments(route)
+        current = segment(route)
+        parents = parents(route).map { |parent| parent_segment(parent) }
+        [parents, current].join
+      end
+
+      def parent_segment(parent)
+        if trailer?(parent)
+          INDENT
+        else
+          REGULAR_SEGMENT
+        end
+      end
+
+      def segment(route)
+        if trailer?(route)
+          CORNER_SEGMENT
+        else
+          JUNCTION_SEGMENT
+        end
+      end
+
+      def label(route)
+        [highlight_matcher(route, matcher_label(route)),
+         highlight_options(route, options(route)),
+         highlight_options(route, params(route))].compact.join(" ")
+      end
+
+      def highlight_matcher(route, string)
+        if path_finder.path.include?(route)
+          set_color(string, :green, :bold)
+        elsif route.matcher.match(url)
+          set_color(string, :green)
+        else
+          set_color(string, :red)
+        end
+      end
+
+      def highlight_options(route, string)
+        return string unless path_finder.path.include?(route)
+
+        set_color(string, :green, :bold)
+      end
+
+      def matcher_label(route)
+        return "Target" if route.is_a?(Wayfarer::Routing::TargetRoute)
+
+        route.matcher.class.name.demodulize
+      end
+
+      def options(route)
+        return "" if route.is_a?(Wayfarer::Routing::RootRoute)
+
+        case (matcher = route.matcher)
+        when Wayfarer::Routing::Matchers::Host then matcher.host
+        when Wayfarer::Routing::Matchers::Path then matcher.path
+        when Wayfarer::Routing::Matchers::Query then matcher.fields
+        when Wayfarer::Routing::Matchers::Custom then "##{route.action}"
+        when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
+        when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
+        end
+      end
+
+      def params(route)
+        params = route.matcher.params(url)
+        "=> #{params.symbolize_keys}" if params.any?
+      end
+
+    private
+
+      def parents(route, current = [])
+        return current unless route.parent
+
+        parents(route.parent, [route.parent, *current])
+      end
+
+      def trailer?(route)
+        return true unless route.parent
+
+        route.parent.children.last == route
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/runner.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Runner < Thor
+      def self.exit_on_failure?
+        true
+      end
+
+      def self.loader
+        # TODO: Raises if $PWD/app/jobs does not exist
+
+        @loader ||= Zeitwerk::Loader.new.tap do |load|
+          load.inflector = Zeitwerk::GemInflector.new(__FILE__)
+          load.push_dir("#{Dir.pwd}/app/jobs")
+        end
+      end
+
+      desc "version", "Print version"
+      def version
+        say Wayfarer::VERSION::STRING
+      end
+
+      desc "route", "Routing"
+      subcommand "route", Wayfarer::CLI::Route
+
+      desc "generate", "Generators"
+      subcommand "generate", Wayfarer::CLI::Generate
+
+      desc "job", "Jobs"
+      subcommand "job", Wayfarer::CLI::Job
+    end
+  end
+end

data/lib/wayfarer/cli/templates/Gemfile.tt

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "wayfarer", <%= Wayfarer::VERSION::STRING.inspect %>

data/lib/wayfarer/cli/templates/job.rb.tt

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class <%= @name.camelize %> < Wayfarer::Base
+  route { to :index }
+
+  retry_on ConnectionPool::TimeoutError, attempts: 3
+
+  def index
+  end
+end

data/lib/wayfarer/config/capybara.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Capybara = Struct.new(driver: {
+                            env_key: "WAYFARER_CAPYBARA_DRIVER",
+                            type: Symbol
+                          })
+  end
+end

data/lib/wayfarer/config/ferrum.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Ferrum = Struct.new(options: {
+                          env_key: "WAYFARER_FERRUM_OPTIONS",
+                          type: Hash,
+                          default: {}
+                        })
+  end
+end

data/lib/wayfarer/config/networking.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Networking = Struct.new(agent: {
+                              env_key: "WAYFARER_NETWORK_AGENT",
+                              type: Symbol,
+                              default: :http
+                            },
+                            pool_size: {
+                              env_key: "WAYFARER_NETWORK_POOL_SIZE",
+                              type: Integer,
+                              default: 1
+                            },
+                            pool_timeout: {
+                              env_key: "WAYFARER_NETWORK_POOL_TIMEOUT",
+                              type: Integer,
+                              default: 10
+                            },
+                            http_headers: {
+                              env_key: "WAYFARER_NETWORK_HTTP_HEADERS",
+                              type: Hash,
+                              default: {}
+                            })
+  end
+end

data/lib/wayfarer/config/redis.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Redis = Struct.new(url: {
+                         env_key: "WAYFARER_REDIS_URL",
+                         type: String,
+                         default: "redis://localhost:6379"
+                       },
+                       factory: {
+                         default: ->(redis) { ::Redis.new(url: redis.url) }
+                       })
+  end
+end

data/lib/wayfarer/config/root.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Root = Struct.new(ferrum: { default: Wayfarer::Config::Ferrum.new },
+                      network: { default: Wayfarer::Config::Networking.new },
+                      redis: { default: Wayfarer::Config::Redis.new },
+                      selenium: { default: Wayfarer::Config::Selenium.new },
+                      capybara: { default: Wayfarer::Config::Capybara.new })
+  end
+end

data/lib/wayfarer/config/selenium.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    Selenium = Struct.new(driver: {
+                            env_key: "WAYFARER_SELENIUM_DRIVER",
+                            type: Symbol,
+                            default: :chrome
+                          },
+                          options: {
+                            env_key: "WAYFARER_SELENIUM_OPTIONS",
+                            type: Hash,
+                            default: {}
+                          },
+                          client_timeout: {
+                            env_key: "WAYFARER_SELENIUM_CLIENT_TIMEOUT",
+                            type: Integer,
+                            default: 60 # seconds
+                          })
+  end
+end

data/lib/wayfarer/config/strconv.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Config
+    module Strconv
+    module_function
+
+      def parse(str, type = nil)
+        return primitive(str) unless type
+
+        case type.name
+        when "Hash" then hash(str)
+        when "Array" then array(str)
+        when "Symbol" then str.to_sym
+        when "Integer" then Integer(str)
+        else str
+        end
+      end
+
+      def hash(str)
+        array(str).reduce({}) do |acc, pair|
+          k, v = pair.split(":", 2)
+          next acc unless k && v
+
+          acc.merge({ parse(k, Symbol) => primitive(v) })
+        end
+      end
+
+      def array(str)
+        str.split(",").map(&:strip)
+      end
+
+      def primitive(str)
+        return true if str == "true"
+        return false if str == "false"
+
+        begin
+          parse(str, Integer)
+        rescue StandardError
+          str
+        end
+      end
+    end
+  end
+end