wayfarer 0.0.3 → 0.4.0

data/docs/reference/environment_variables.md

@@ -0,0 +1,85 @@
+# Environment Variables
+
+## String formats
+
+Environment variable values can be parsed to Hash or Array at runtime
+with the following syntaxes:
+
+* Hash: Variable string `a:1,b:2,c:3` parses to `{a:1, b:2, c:3}` at runtime
+* Array: Variable string `a,b,c` parses to `[:a, :b, :c]` at runtime
+
+## Variables
+
+### `WAYFARER_ADAPTER`
+:   Either `ferrum`, `selenium` or `net_http`.
+
+    * Type: String
+    * Key: `config.adapter`
+    * Default value: `:net_http`
+
+### `WAYFARER_POOL_SIZE`
+:   Number of network adapters to maintain.
+
+    * Type: Integer
+    * Key: `config.pool_size`
+    * Default value: `1`
+
+### `WAYFARER_POOL_TIMEOUT`
+:   How long a network adapter may remain checked out until the owning job
+    fails.
+
+    * Type: Integer
+    * Key: `config.adapter_pool_timeout`
+    * Default value: `1`
+
+---
+
+### `WAYFARER_FERRUM_OPTIONS`
+:   Key/value options passed to `Ferrum::Browser.new`.
+
+    * Type: Hash
+    * Key: `config.ferrum_options`
+    * Default value: `{}`
+
+---
+
+### `WAYFARER_SELENIUM_ARGV`
+:   Argument list passed to `Selenium::WebDriver.for`.
+
+    * Type: Array
+    * Key: `config.selenium_argv`
+    * Default value: `[]`
+
+## Examples
+
+!!! example "Foobar"
+
+    For example, to run Google Chrome in foreground with Ferrum:
+
+    ```
+    Wayfarer.config.adapter = :ferrum
+    Wayfarer.ferrum_options = { headless: false, url: "http://chrome:3000" }
+    ```
+
+
+!!! example "Run Google Chrome in foreground with Ferrum"
+
+    To construct the 
+
+    ```
+    WAYFARER_FERRUM_OPTIONS=headless:false,url:http://chrome:3000
+    ```
+
+---
+
+### `WAYFARER_HTTP_HEADERS`
+:   HTTP request headers used when retrieving pages.
+
+    * Type: Hash
+    * Key: `config.http_headers`
+    * Default value: `{}`
+    
+    !!! attention "Partial support"
+
+        Selenium does not support configuring HTTP request headers.
+

data/lib/wayfarer/base.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Base < ActiveJob::Base
+    include Wayfarer::Middleware::Worker
+    extend Forwardable
+
+    class << self
+      def after_batch_callbacks
+        @after_batch_callbacks ||= []
+      end
+
+      def after_batch(&block)
+        after_batch_callbacks.push(block)
+      end
+
+      def run_after_batch_callbacks
+        after_batch_callbacks.each(&:call)
+      end
+    end
+
+    after_enqueue do |job|
+      task = job.arguments.first
+      task.counter.increment
+    end
+
+    after_perform { |job| GC.new(job).run }
+
+    rescue_from(StandardError) { GC.new(self).run }
+
+    def self.retry_on(*argv)
+      super(*argv) do |job, error|
+        GC.new(job).run
+        yield job, error if block_given?
+      end
+    end
+
+    def self.discard_on(*argv)
+      super(*argv) do |job, error|
+        GC.new(job).run
+        yield job, error if block_given?
+      end
+    end
+
+    def self.crawl_later(url, batch: SecureRandom.uuid)
+      perform_later(Task.new(url, batch))
+    end
+
+    def retry_job(...)
+      super(...) # increments the counter by re-enqueuing the job
+      task = arguments.first
+      task.counter.decrement
+    end
+
+    def perform(task)
+      task.job = self
+      chain.call(task)
+    end
+  end
+end

data/lib/wayfarer/cli/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Base < Thor
+      include Thor::Actions
+
+      def self.source_root
+        File.expand_path("templates", __dir__)
+      end
+
+    private
+
+      def mock_redis
+        Wayfarer.config.redis_factory = -> { MockRedis.new }
+      end
+
+      def load_environment
+        Wayfarer::CLI::Runner.loader.setup
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/generate.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Generate < Base
+      desc "project NAME",
+           "Generate new project directory NAME"
+      def project(name)
+        @name = name
+
+        empty_directory name
+        template "Gemfile.tt", "#{name}/Gemfile"
+        template "job.rb.tt", "#{name}/app/jobs/#{name}.rb"
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/job.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Job < Base
+      desc "perform JOB URL",
+           "Perform JOB with URL"
+      option :mock_redis, type: :boolean
+      option :batch, type: :string, default: SecureRandom.uuid
+      def perform(job, url)
+        load_environment
+        mock_redis if options[:mock_redis]
+
+        url = URI(url)
+        job = job.classify.constantize.new
+        task = Wayfarer::Task.new(url, "tmp")
+        job.arguments.push(task)
+        job.perform(task)
+        GC.new(job).run
+      end
+
+      desc "enqueue JOB URL",
+           "Enqueue JOB with URL"
+      option :batch, type: :string, default: SecureRandom.uuid
+      def enqueue(job, url)
+        load_environment
+        mock_redis if options[:mock_redis]
+
+        url = URI(url)
+        job = job.classify.constantize
+        job.crawl_later(url, batch: options[:batch])
+      end
+
+      desc "execute JOB URL",
+           "Execute JOB with async adapter"
+      option :mock_redis, type: :boolean
+      option :batch, type: :string, default: SecureRandom.uuid
+      option :min_threads, type: :numeric, default: 1
+      option :max_threads, type: :numeric, default: 1
+      def execute(job, url)
+        load_environment
+        mock_redis if options[:mock_redis]
+
+        url = URI(url)
+        job = job.classify.constantize
+
+        job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
+                                                                       max_threads: options[:max_threads])
+        scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
+        executor = scheduler.instance_variable_get(:@async_executor)
+
+        job.crawl_later(url, batch: options[:batch])
+
+        sleep(1) while executor.scheduled_task_count > executor.completed_task_count
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/route.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# # frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Route < Base
+      desc "result JOB URL",
+           "Invoke JOB's router with URL"
+      def result(job, url)
+        load_environment
+        url = URI(url)
+        job = job.classify.constantize
+        puts Wayfarer::Routing::PathFinder.result(job.route, url)
+      end
+
+      desc "tree JOB URL",
+           "Visualize JOB's routing tree for URL"
+      def tree(job, url)
+        load_environment
+        url = URI(url)
+        job = job.classify.constantize
+        Wayfarer::CLI::RoutePrinter.print(job.route, url)
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/route_printer.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class RoutePrinter < Thor::Shell::Color
+      attr_reader :url,
+                  :path_finder,
+                  :output
+
+      INDENT           = "   "
+      REGULAR_SEGMENT  = "│  "
+      JUNCTION_SEGMENT = "├──"
+      CORNER_SEGMENT   = "└──"
+
+      def self.print(route, url)
+        route.accept(new(url))
+      end
+
+      def initialize(url)
+        @url = url
+        @path_finder = Wayfarer::Routing::PathFinder.new(url)
+        super()
+      end
+
+      def visit(route)
+        route.accept(path_finder) unless route.parent
+        return true if route.is_a?(Wayfarer::Routing::RootRoute)
+
+        puts [segments(route), label(route)].join("")[3..]
+        true
+      end
+
+      def segments(route)
+        current = segment(route)
+        parents = parents(route).map { |parent| parent_segment(parent) }
+        [parents, current].join
+      end
+
+      def parent_segment(parent)
+        if trailer?(parent)
+          INDENT
+        else
+          REGULAR_SEGMENT
+        end
+      end
+
+      def segment(route)
+        if trailer?(route)
+          CORNER_SEGMENT
+        else
+          JUNCTION_SEGMENT
+        end
+      end
+
+      def label(route)
+        [highlight_matcher(route, matcher_label(route)),
+         highlight_options(route, options(route)),
+         highlight_options(route, params(route))].compact.join(" ")
+      end
+
+      def highlight_matcher(route, string)
+        if path_finder.path.include?(route)
+          set_color(string, :green, :bold)
+        elsif route.matcher.match(url)
+          set_color(string, :green)
+        else
+          set_color(string, :red)
+        end
+      end
+
+      def highlight_options(route, string)
+        return string unless path_finder.path.include?(route)
+
+        set_color(string, :green, :bold)
+      end
+
+      def matcher_label(route)
+        return "Target" if route.is_a?(Wayfarer::Routing::TargetRoute)
+
+        route.matcher.class.name.demodulize.delete_suffix("Matcher")
+      end
+
+      def options(route)
+        return "" if route.is_a?(Wayfarer::Routing::RootRoute)
+
+        case (matcher = route.matcher)
+        when Wayfarer::Routing::HostMatcher then matcher.host
+        when Wayfarer::Routing::PathMatcher then matcher.path
+        when Wayfarer::Routing::QueryMatcher then matcher.fields
+        when Wayfarer::Routing::CustomMatcher then "##{route.action}"
+        when Wayfarer::Routing::SchemeMatcher then matcher.scheme
+        when Wayfarer::Routing::SuffixMatcher then matcher.suffix
+        end
+      end
+
+      def params(route)
+        params = route.matcher.params(url)
+        "=> #{params.symbolize_keys}" if params.any?
+      end
+
+    private
+
+      def parents(route, current = [])
+        return current unless route.parent
+
+        parents(route.parent, [route.parent, *current])
+      end
+
+      def trailer?(route)
+        return true unless route.parent
+
+        route.parent.children.last == route
+      end
+    end
+  end
+end

data/lib/wayfarer/cli/runner.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module CLI
+    class Runner < Thor
+      def self.exit_on_failure?
+        true
+      end
+
+      def self.loader
+        # TODO: Raises if $PWD/app/jobs does not exist
+
+        @loader ||= Zeitwerk::Loader.new.tap do |load|
+          load.inflector = Zeitwerk::GemInflector.new(__FILE__)
+          load.push_dir("#{Dir.pwd}/app/jobs")
+        end
+      end
+
+      desc "version", "Print version"
+      def version
+        say Wayfarer::VERSION::STRING
+      end
+
+      desc "route", "Routing"
+      subcommand "route", Wayfarer::CLI::Route
+
+      desc "generate", "Generators"
+      subcommand "generate", Wayfarer::CLI::Generate
+
+      desc "job", "Jobs"
+      subcommand "job", Wayfarer::CLI::Job
+    end
+  end
+end

data/lib/wayfarer/cli/templates/Gemfile.tt

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "wayfarer", <%= Wayfarer::VERSION::STRING.inspect %>

data/lib/wayfarer/cli/templates/job.rb.tt

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class <%= @name.camelize %> < Wayfarer::Base
+  route.to :index
+
+  def index
+  end
+end

data/lib/wayfarer/config.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  # rubocop:disable Lint/ConstantDefinitionInBlock
+  Config = Struct.new(:adapter,
+                      :adapter_pool_size,
+                      :adapter_pool_timeout,
+                      :ferrum_options,
+                      :selenium_argv,
+                      :redis_url,
+                      :redis_factory,
+                      :http_headers) do
+    DEFAULTS = {
+      adapter: :net_http,
+      adapter_pool_size: 3,
+      adapter_pool_timeout: ::ConnectionPool::DEFAULTS[:timeout],
+      ferrum_options: {},
+      selenium_argv: %i[chrome],
+      redis_url: "redis://localhost:6379",
+      redis_factory: -> { ::Redis.new(url: Wayfarer.config.redis_url) },
+      http_headers: {}
+    }.freeze
+
+    def self.default
+      new(*DEFAULTS.values)
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def self.from_environment(env = ENV)
+      default.tap do |config|
+        config.adapter = env["WAYFARER_ADAPTER"].to_sym if env.key?("WAYFARER_ADAPTER")
+        config.adapter_pool_size = cast(env["WAYFARER_POOL_SIZE"]) if env.key?("WAYFARER_POOL_SIZE")
+        config.adapter_pool_timeout = cast(env["WAYFARER_POOL_TIMEOUT"]) if env.key?("WAYFARER_POOL_TIMEOUT")
+        config.ferrum_options = parse_hash(env["WAYFARER_FERRUM_OPTIONS"]) if env.key?("WAYFARER_FERRUM_OPTIONS")
+        config.selenium_argv = parse_array(env["WAYFARER_SELENIUM_ARGV"]) if env.key?("WAYFARER_SELENIUM_ARGV")
+        config.redis_url = env["WAYFARER_REDIS_URL"] if env.key?("WAYFARER_REDIS_URL")
+        config.http_headers = parse_hash(env["WAYFARER_HTTP_HEADERS"]) if env.key?("WAYFARER_HTTP_HEADERS")
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    def self.parse_hash(str)
+      parse_array(str).reduce({}) do |acc, pair|
+        k, v = pair.split(":", 2)
+        next acc unless k && v
+
+        acc.merge({ k.to_sym => cast(v) })
+      end
+    end
+
+    def self.parse_array(str)
+      str.split(",").map(&:strip)
+    end
+
+    def self.cast(str)
+      return true if str == "true"
+      return false if str == "false"
+
+      begin
+        Integer(str)
+      rescue StandardError
+        str
+      end
+    end
+  end
+  # rubocop:enable Lint/ConstantDefinitionInBlock
+end

data/lib/wayfarer/gc.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  GC = Struct.new(:job) do
+    def run
+      return unless task.counter.decrement <= 0
+
+      task.barrier.reset!
+      task.counter.reset!
+      job.class.run_after_batch_callbacks
+    end
+
+  private
+
+    def task
+      job.arguments.first
+    end
+  end
+end

data/lib/wayfarer/middleware/chain.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    Chain = Struct.new(:middlewares) do
+      def call(task)
+        traverse = lambda do |i|
+          if middlewares[i]
+            middlewares[i].call(task) { traverse.call(i + 1) }
+          elsif block_given?
+            yield
+          end
+        end
+
+        traverse.call(0)
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/dedup.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Dedup
+      def call(task)
+        return if task.barrier.seen?(task.url)
+
+        begin
+          yield if block_given?
+        rescue StandardError => e
+          task.barrier.unsee(task.url)
+          raise e
+        end
+
+        staged_urls = task.metadata.staged_urls
+        return if staged_urls.none?
+
+        inclusion = task.barrier.peek(staged_urls.to_a)
+        unseen = staged_urls.zip(inclusion).reject { |_, seen| seen }.map(&:first)
+        task.metadata.staged_urls = SortedSet.new(unseen)
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/fetch.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Fetch
+      include Wayfarer::Middleware::Stage::API
+
+      attr_reader :pool
+      attr_accessor :task
+
+      def initialize
+        @pool = Wayfarer::Networking::Pool.instance
+      end
+
+      def call(task)
+        self.task = task
+
+        pool.with do |adapter|
+          task.metadata.adapter = adapter
+
+          case result = adapter.fetch(task.url)
+          when Networking::Result::Redirect
+            stage(result.redirect_url)
+          when Networking::Result::Success
+            task.metadata.page = result.page
+            yield if block_given?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/normalize.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Normalize
+      def call(task)
+        yield if block_given?
+
+        task.metadata.staged_urls = SortedSet.new(normalized_urls(task).compact)
+      end
+
+    private
+
+      def normalized_urls(task)
+        task.metadata.staged_urls.map(&method(:normalize))
+      end
+
+      def normalize(url)
+        NormalizeUrl.process(url)
+      rescue NormalizeUrl::InvalidURIError
+        nil
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/router.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Router
+      def call(task)
+        route = task.job.class.route
+
+        case result = route.invoke(URI(task.url))
+        when Routing::Result::Mismatch
+          return
+        when Routing::Result::Match
+          task.metadata.action = result.action
+          task.metadata.params = result.params
+        end
+
+        yield if block_given?
+      end
+    end
+  end
+end

data/lib/wayfarer/middleware/stage.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Middleware
+    class Stage
+      module API
+        def stage(urls)
+          Array.wrap(urls).each { |url| task.metadata.staged_urls.add(url) }
+        end
+      end
+
+      def call(task)
+        task.metadata.staged_urls = SortedSet.new
+
+        yield if block_given?
+
+        task.metadata.staged_urls.each do |url|
+          task.job.class.crawl_later(url, batch: task.batch)
+        end
+      end
+    end
+  end
+end