tobox 0.1.0

data/lib/tobox/cli.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "logger"
+require "optparse"
+require "uri"
+require_relative "../tobox"
+
+module Tobox
+  class CLI
+    def self.run(args = ARGV)
+      new(args).run
+    end
+
+    def initialize(args)
+      @options = parse(args)
+    end
+
+    def run
+      options = @options
+      logger = Logger.new($stderr)
+
+      config = Configuration.new do |c|
+        c.logger(logger)
+        c.instance_eval(File.read(options.fetch(:config_file)), options.fetch(:config_file), 1)
+      end
+
+      # boot
+      options.fetch(:require).each(&method(:require))
+
+      # signals
+      pipe_read, pipe_write = IO.pipe
+      %w[INT TERM].each do |sig|
+        old_handler = Signal.trap(sig) do
+          if old_handler.respond_to?(:call)
+            begin
+              old_handler.call
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              puts ["Error in #{sig} handler", e].inspect
+            end
+          end
+          pipe_write.puts(sig)
+        end
+      rescue ArgumentError
+        puts "Signal #{sig} not supported"
+      end
+
+      app = Tobox::Application.new(config)
+
+      begin
+        app.start
+
+        logger.info "Running tobox-#{Tobox::VERSION} (#{RUBY_DESCRIPTION})"
+        logger.info "workers=#{config[:concurrency]}"
+        logger.info "Press Ctrl-C to stop"
+
+        while pipe_read.wait_readable
+          signal = pipe_read.gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        logger.info "Shutting down..."
+        app.stop
+        logger.info "Down!"
+        exit(0)
+      end
+    end
+
+    private
+
+    def parse(args)
+      opts = {
+        require: []
+      }
+      parser = OptionParser.new do |o|
+        o.on "-C", "--config PATH", "path to tobox .rb config file" do |arg|
+          arg = File.join(arg, "tobox.rb") if File.directory?(arg)
+          raise ArgumentError, "no such file #{arg}" unless File.exist?(arg)
+
+          opts[:config_file] = arg
+        end
+
+        o.on "-r", "--require [PATH|DIR]", "Location of application with files to require" do |arg|
+          requires = (opts[:require] ||= [])
+          if File.directory?(arg)
+            requires.concat(Dir.glob(File.join("**", "*.rb")))
+          else
+            raise ArgumentError, "no such file #{arg}" unless File.exist?(arg)
+
+            requires << arg
+          end
+        end
+
+        o.on "-d", "--database-uri DATABASE_URI", String, "location of the database with the outbox table" do |arg|
+          opts[:database_uri] = URI(arg)
+        end
+
+        o.on "-t", "--table TABLENAME", "(optional) name of the outbox database table" do |arg|
+          opts[:table] = arg
+        end
+
+        o.on "-c", "--concurrency INT", Integer, "processor threads to use" do |arg|
+          raise ArgumentError, "must be positive" unless arg.positive?
+
+          opts[:concurrency] = arg
+        end
+
+        o.on "-g", "--tag TAG", "Process tag for procline" do |arg|
+          opts[:tag] = arg
+        end
+
+        o.on "-t", "--shutdown-timeout NUM", Integer, "Shutdown timeout (in seconds)" do |arg|
+          raise ArgumentError, "must be positive" unless arg.positive?
+
+          opts[:shutdown_timeout] = arg
+        end
+
+        o.on "--verbose", "Print more verbose output" do |arg|
+          opts[:verbose] = arg
+        end
+
+        o.on "-v", "--version", "Print version and exit" do |_arg|
+          puts "Tobox #{Tobox::VERSION}"
+          exit(0)
+        end
+      end
+
+      parser.banner = "tobox [options]"
+      parser.on_tail "-h", "--help", "Show help" do
+        $stdout.puts parser
+        exit(0)
+      end
+      parser.parse(args)
+      opts
+    end
+
+    def handle_signal(sig)
+      case sig
+      when "INT", "TERM"
+        raise Interrupt
+      else
+        warn "#{sig} is unsupported"
+      end
+    end
+  end
+end

data/lib/tobox/configuration.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "logger"
+require "forwardable"
+
+module Tobox
+  class Configuration
+    extend Forwardable
+
+    attr_reader :handlers, :lifecycle_events, :arguments_handler, :default_logger
+
+    def_delegator :@config, :[]
+
+    DEFAULT_CONFIGURATION = {
+      environment: ENV.fetch("APP_ENV", "development"),
+      logger: nil,
+      log_level: nil,
+      database_uri: nil,
+      table: :outbox,
+      max_attempts: 10,
+      exponential_retry_factor: 4,
+      wait_for_events_delay: 5,
+      shutdown_timeout: 10,
+      concurrency: 4, # TODO: CPU count
+      worker: :thread
+    }.freeze
+
+    def initialize(name = nil, &block)
+      @name = name
+      @config = DEFAULT_CONFIGURATION.dup
+
+      @lifecycle_events = {}
+      @handlers = {}
+      @message_to_arguments = nil
+      @plugins = []
+
+      if block
+        case block.arity
+        when 0
+          instance_exec(&block)
+        when 1
+          yield(self)
+        else
+          raise Error, "configuration does not support blocks with more than one variable"
+        end
+      end
+
+      env = @config[:environment]
+      @default_logger = @config[:logger] || Logger.new(STDERR) # rubocop:disable Style/GlobalStdStream
+      @default_logger.level = @config[:log_level] || (env == "production" ? Logger::INFO : Logger::DEBUG)
+
+      freeze
+    end
+
+    def on(event, &callback)
+      (@handlers[event.to_sym] ||= []) << callback
+      self
+    end
+
+    def on_before_event(&callback)
+      (@lifecycle_events[:before_event] ||= []) << callback
+      self
+    end
+
+    def on_after_event(&callback)
+      (@lifecycle_events[:after_event] ||= []) << callback
+      self
+    end
+
+    def on_error_event(&callback)
+      (@lifecycle_events[:error_event] ||= []) << callback
+      self
+    end
+
+    def message_to_arguments(&callback)
+      @arguments_handler = callback
+      self
+    end
+
+    def plugin(plugin, _options = nil, &block)
+      raise Error, "Cannot add a plugin to a frozen config" if frozen?
+
+      plugin = Plugins.load_plugin(plugin) if plugin.is_a?(Symbol)
+
+      return if @plugins.include?(plugin)
+
+      @plugins << plugin
+      plugin.load_dependencies(self, &block) if plugin.respond_to?(:load_dependencies)
+
+      extend(plugin::ConfigurationMethods) if defined?(plugin::ConfigurationMethods)
+
+      plugin.configure(self, &block) if plugin.respond_to?(:configure)
+    end
+
+    def freeze
+      @name.freeze
+      @config.each_value(&:freeze).freeze
+      @handlers.each_value(&:freeze).freeze
+      @lifecycle_events.each_value(&:freeze).freeze
+      @plugins.freeze
+      super
+    end
+
+    private
+
+    def method_missing(meth, *args, &block)
+      if DEFAULT_CONFIGURATION.key?(meth) && args.size == 1
+        @config[meth] = args.first
+      elsif /\Aon_(.*)\z/.match(meth) && args.size.zero?
+        on(Regexp.last_match(1).to_sym, &block)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(meth, *args)
+      super(meth, *args) ||
+        DEFAULT_CONFIGURATION.key?(meth) ||
+        /\Aon_(.*)\z/.match(meth)
+    end
+  end
+end

data/lib/tobox/fetcher.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Tobox
+  class Fetcher
+    def initialize(configuration)
+      @configuration = configuration
+
+      @logger = @configuration.default_logger
+
+      database_uri = @configuration[:database_uri]
+      @db = database_uri ? Sequel.connect(database_uri.to_s) : Sequel::DATABASES.first
+      raise Error, "no database found" unless @db
+
+      @db.extension :date_arithmetic
+
+      @db.loggers << @logger unless @configuration[:environment] == "production"
+
+      @table = configuration[:table]
+      @exponential_retry_factor = configuration[:exponential_retry_factor]
+
+      max_attempts = configuration[:max_attempts]
+
+      @ds = @db[@table]
+
+      run_at_conds = [
+        { Sequel[@table][:run_at] => nil },
+        (Sequel.expr(Sequel[@table][:run_at]) < Sequel::CURRENT_TIMESTAMP)
+      ].reduce { |agg, cond| Sequel.expr(agg) | Sequel.expr(cond) }
+
+      @pick_next_sql = @ds.where(Sequel[@table][:attempts] < max_attempts) # filter out exhausted attempts
+                          .where(run_at_conds)
+                          .order(Sequel.desc(:run_at, nulls: :first), :id)
+                          .for_update
+                          .skip_locked
+                          .limit(1)
+
+      @before_event_handlers = Array(@configuration.lifecycle_events[:before_event])
+      @after_event_handlers = Array(@configuration.lifecycle_events[:after_event])
+      @error_event_handlers = Array(@configuration.lifecycle_events[:error_event])
+    end
+
+    def fetch_events(&blk)
+      num_events = 0
+      @db.transaction do
+        event_ids = @pick_next_sql.select_map(:id) # lock starts here
+
+        events = nil
+        error = nil
+        unless event_ids.empty?
+          @db.transaction(savepoint: true) do
+            events = @ds.where(id: event_ids).returning.delete
+
+            if blk
+              num_events = events.size
+
+              events.each do |ev|
+                ev[:metadata] = JSON.parse(ev[:metadata].to_s) if ev[:metadata]
+                handle_before_event(ev)
+                yield(to_message(ev))
+              rescue StandardError => e
+                error = e
+                raise Sequel::Rollback
+              end
+            else
+              events.map!(&method(:to_message))
+            end
+          end
+        end
+
+        return blk ? 0 : [] if events.nil?
+
+        return events unless blk
+
+        if events
+          events.each do |event|
+            if error
+              event.merge!(mark_as_error(event, error))
+              handle_error_event(event, error)
+            else
+              handle_after_event(event)
+            end
+          end
+        end
+      end
+
+      num_events
+    end
+
+    private
+
+    def mark_as_error(event, error)
+      @ds.where(id: event[:id]).returning.update(
+        attempts: Sequel[@table][:attempts] + 1,
+        run_at: Sequel.date_add(Sequel::CURRENT_TIMESTAMP,
+                                seconds: event[:attempts] + (1**@exponential_retry_factor)),
+        # run_at: Sequel.date_add(Sequel::CURRENT_TIMESTAMP,
+        #                         seconds: Sequel.function(:POWER, Sequel[@table][:attempts] + 1,  4)),
+        last_error: "#{error.message}\n#{error.backtrace.join("\n")}"
+      ).first
+    end
+
+    def to_message(event)
+      {
+        id: event[:id],
+        type: event[:type],
+        before: (JSON.parse(event[:data_before].to_s) if event[:data_before]),
+        after: (JSON.parse(event[:data_after].to_s) if event[:data_after]),
+        at: event[:created_at]
+      }
+    end
+
+    def handle_before_event(event)
+      @logger.debug { "outbox event (type: \"#{event[:type]}\", attempts: #{event[:attempts]}) starting..." }
+      @before_event_handlers.each do |hd|
+        hd.call(event)
+      end
+    end
+
+    def handle_after_event(event)
+      @logger.debug { "outbox event (type: \"#{event[:type]}\", attempts: #{event[:attempts]}) completed" }
+      @after_event_handlers.each do |hd|
+        hd.call(event)
+      end
+    end
+
+    def handle_error_event(event, error)
+      @logger.error do
+        "outbox event (type: \"#{event[:type]}\", attempts: #{event[:attempts]}) failed with error\n" \
+          "#{error.class}: #{error.message}\n" \
+          "#{error.backtrace.join("\n")}"
+      end
+      @error_event_handlers.each do |hd|
+        hd.call(event, error)
+      end
+    end
+  end
+end

data/lib/tobox/plugins/datadog/configuration.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "datadog/tracing/contrib/configuration/settings"
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Tobox
+        module Configuration
+          class Settings < Contrib::Configuration::Settings
+            option :enabled do |o|
+              o.default { env_to_bool("DD_TRACE_SIDEKIQ_ENABLED", true) }
+              o.lazy
+            end
+
+            option :analytics_enabled do |o|
+              o.default { env_to_bool("DD_TRACE_SIDEKIQ_ANALYTICS_ENABLED", false) }
+              o.lazy
+            end
+
+            option :analytics_sample_rate do |o|
+              o.default { env_to_float("DD_TRACE_SIDEKIQ_ANALYTICS_SAMPLE_RATE", 1.0) }
+              o.lazy
+            end
+
+            option :service_name
+            option :error_handler, default: SpanOperation::Events::DEFAULT_ON_ERROR
+            option :distributed_tracing, default: false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tobox/plugins/datadog/integration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "datadog/tracing/contrib/integration"
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Tobox
+        class Integration
+          include Contrib::Integration
+
+          MINIMUM_VERSION = Gem::Version.new("0.1.0")
+
+          register_as :tobox
+
+          def self.version
+            Gem.loaded_specs["tobox"] && Gem.loaded_specs["tobox"].version
+          end
+
+          def self.loaded?
+            !defined?(::Tobox).nil?
+          end
+
+          def self.compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tobox/plugins/datadog/patcher.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "datadog/tracing/contrib/patcher"
+
+module Datadog
+  module Tracing
+    module Contrib
+      module Tobox
+        module Patcher
+          include Contrib::Patcher
+
+          module_function
+
+          def target_version
+            Integration.version
+          end
+
+          def patch
+            # server-patches provided by plugin(:sidekiq)
+            # TODO: use this once we have a producer side
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tobox/plugins/datadog.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative "datadog/configuration"
+require_relative "datadog/integration"
+require_relative "datadog/patcher"
+
+module Tobox
+  module Plugins
+    module Datadog
+      class EventHandler
+        def initialize(config)
+          @config = config
+          @db_table = @config[:table]
+        end
+
+        def on_start(event)
+          datadog_config = ::Datadog.configuration.tracing[:tobox]
+          service = datadog_config[:service_name]
+          error_handler = datadog_config[:error_handler]
+
+          analytics_enabled = datadog_config[:analytics_enabled]
+          analytics_sample_rate = datadog_config[:analytics_sample_rate]
+          distributed_tracing = datadog_config[:distributed_tracing]
+
+          resource = event[:type]
+
+          if (metadata = event[:metadata])
+            previous_span = metadata["datadog-parent-id"]
+
+            if distributed_tracing && previous_span
+              trace_digest = ::Datadog::Tracing::TraceDigest.new(
+                span_id: previous_span,
+                trace_id: event[:metadata]["datadog-trace-id"],
+                trace_sampling_priority: event[:metadata]["datadog-sampling-priority"],
+                trace_origin: event[:metadata]["datadog-origin"]
+              )
+              ::Datadog::Tracing.continue_trace!(trace_digest)
+            end
+          end
+
+          span = ::Datadog::Tracing.trace(
+            "tobox.event",
+            service: service,
+            span_type: ::Datadog::Tracing::Metadata::Ext::AppTypes::TYPE_WORKER,
+            on_error: error_handler
+          )
+          span.resource = resource
+
+          span.set_tag(::Datadog::Tracing::Metadata::Ext::TAG_COMPONENT, "tobox")
+          span.set_tag(::Datadog::Tracing::Metadata::Ext::TAG_OPERATION, "event")
+
+          if ::Datadog::Tracing::Contrib::Analytics.enabled?(analytics_enabled)
+            ::Datadog::Tracing::Contrib::Analytics.set_sample_rate(span, analytics_sample_rate)
+          end
+
+          # Measure service stats
+          ::Datadog::Tracing::Contrib::Analytics.set_measured(span)
+
+          span.set_tag("tobox.event.id", event[:id])
+          span.set_tag("tobox.event.type", event[:type])
+          span.set_tag("tobox.event.retry", event[:attempts])
+          span.set_tag("tobox.event.table", @db_table)
+          span.set_tag("tobox.event.delay", (Time.now.utc - event[:created_at]).to_f)
+
+          event[:__tobox_event_span] = span
+        end
+
+        def on_finish(event)
+          span = event[:__tobox_event_span]
+
+          return unless span
+
+          span.finish
+        end
+
+        def on_error(event, error)
+          span = event[:__tobox_event_span]
+
+          return unless span
+
+          span.set_error(error)
+          span.finish
+        end
+      end
+
+      class << self
+        def load_dependencies(*)
+          require "uri"
+        end
+
+        def configure(config)
+          event_handler = EventHandler.new(config)
+          config.on_before_event(&event_handler.method(:on_start))
+          config.on_after_event(&event_handler.method(:on_finish))
+          config.on_error_event(&event_handler.method(:on_error))
+        end
+      end
+    end
+
+    register_plugin :datadog, Datadog
+  end
+end