upperkut 1.0.3 → 1.0.4

data/examples/scheduled_worker.rb

@@ -0,0 +1,19 @@
+require_relative '../lib/upperkut/worker'
+require_relative '../lib/upperkut/strategies/scheduled_queue'
+
+class ScheduledWorker
+  include Upperkut::Worker
+
+  setup_upperkut do |config|
+    config.strategy = Upperkut::Strategies::ScheduledQueue.new(
+      self,
+      batch_size: 200
+    )
+  end
+
+  def perform(items)
+    items.each do |item|
+      puts "event dispatched: #{item.inspect}"
+    end
+  end
+end

data/examples/with_middlewares.rb

@@ -0,0 +1,42 @@
+require_relative '../lib/upperkut/worker'
+require_relative '../lib/upperkut/logging'
+
+class ClientMiddleware
+  def call(worker, items)
+    logger = Upperkut::Logging.logger
+
+    logger.info("inserting worker=#{worker} items=#{items.count}")
+    yield
+    logger.info("inserted worker=#{worker} items=#{items.count}")
+  end
+end
+
+class MyMiddleware
+  def call(worker, items)
+    logger = Upperkut::Logging.logger
+
+    logger.info("performing worker=#{worker} items=#{items.count}")
+    yield
+    logger.info("performed worker=#{worker} items=#{items.count}")
+  end
+end
+
+class WithMiddlewares
+  include Upperkut::Worker
+
+  setup_upperkut do |config|
+    config.server_middlewares do |chain|
+      chain.add MyMiddleware
+    end
+
+    config.client_middlewares do |chain|
+      chain.add ClientMiddleware
+    end
+  end
+
+  def perform(_items)
+    puts 'executing.........'
+    exec_time = rand(80..200)
+    sleep (exec_time.to_f / 1000.to_f)
+  end
+end

data/lib/upperkut/cli.rb

@@ -0,0 +1,100 @@
+require 'optparse'
+require_relative '../upperkut'
+require_relative 'manager'
+require_relative 'logging'
+
+module Upperkut
+  class CLI
+    def initialize(args = ARGV)
+      @options = {}
+      @logger = Upperkut::Logging.logger
+
+      parse_options(args)
+    end
+
+    def start
+      if target_required = @options[:require]
+        if File.directory?(target_required)
+          require 'rails'
+          if ::Rails::VERSION::MAJOR == 4
+            require File.expand_path("#{@options[:require]}/config/application.rb")
+            ::Rails::Application.initializer 'upperkut.eager_load' do
+              ::Rails.application.config.eager_load = true
+            end
+
+            require File.expand_path("#{@options[:require]}/config/environment.rb")
+          else
+            require File.expand_path("#{@options[:require]}/config/environment.rb")
+          end
+        else
+          require target_required
+        end
+      end
+
+      if log_level = @options[:log_level]
+        @logger.level = log_level
+      end
+
+      @options[:logger] = @logger
+
+      manager = Manager.new(@options)
+
+      @logger.info(@options)
+
+      r, w = IO.pipe
+      signals = %w[INT TERM]
+
+      signals.each do |signal|
+        trap signal do
+          w.puts(signal)
+        end
+      end
+
+      begin
+        manager.run
+        while readable_io = IO.select([r])
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        timeout = Integer(ENV['UPPERKUT_TIMEOUT'] || 8)
+        @logger.info(
+          "Stopping managers, wait for #{timeout} seconds and them kill processors"
+        )
+
+        manager.stop
+        sleep(timeout)
+        manager.kill
+        exit(0)
+      end
+    end
+
+    private
+
+    def handle_signal(sig)
+      case sig
+      when 'INT'
+        raise Interrupt
+      when 'TERM'
+        raise Interrupt
+      end
+    end
+
+    def parse_options(args)
+      OptionParser.new do |o|
+        o.on('-w', '--worker WORKER', 'Define worker to be processed') do |arg|
+          @options[:worker] = arg
+        end
+        o.on('-r', '--require FILE', 'Indicate a file to be required') do |arg|
+          @options[:require] = arg
+        end
+        o.on('-c', '--concurrency INT', 'Numbers of threads to spawn') do |arg|
+          @options[:concurrency] = Integer(arg)
+        end
+        o.on('-l', '--log-level LEVEL', 'Log level') do |arg|
+          @options[:log_level] = arg.to_i
+        end
+      end.parse!(args)
+    end
+  end
+end

data/lib/upperkut/core_ext.rb

@@ -0,0 +1,18 @@
+begin
+  require 'active_support/core_ext/string/inflections'
+rescue LoadError
+  unless ''.respond_to?(:constantize)
+    class String
+      def constantize
+        names = split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        constant = Object
+        names.each do |name|
+          constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+        constant
+      end
+    end
+  end
+end

data/lib/upperkut/item.rb

@@ -0,0 +1,22 @@
+require 'securerandom'
+
+module Upperkut
+  class Item
+    attr_reader :id, :body, :enqueued_at
+
+    def initialize(id:, body:, enqueued_at: nil)
+      @id = id
+      @body = body
+      @enqueued_at = enqueued_at || Time.now.utc.to_i
+      @nacked = false
+    end
+
+    def nack
+      @nacked = true
+    end
+
+    def nacked?
+      @nacked
+    end
+  end
+end

data/lib/upperkut/logging.rb

@@ -0,0 +1,36 @@
+require 'logger'
+require 'time'
+require 'socket'
+
+module Upperkut
+  module Logging
+    class DefaultFormatter < Logger::Formatter
+      def call(severity, time, _program_name, message)
+        "upperkut: #{time.utc.iso8601(3)} hostname=#{Socket.gethostname} "\
+        "pid=#{::Process.pid} severity=#{severity} #{format_message(message)}\n"
+      end
+
+      private
+
+      def format_message(message)
+        return "msg=#{message} " unless message.is_a?(Hash)
+
+        message.each_with_object('') do |(k, v), memo|
+          memo << "#{k}=#{v}\s"
+          memo
+        end
+      end
+    end
+
+    def self.initialize_logger
+      logger = Logger.new($stdout)
+      logger.level     = Logger::INFO
+      logger.formatter = DefaultFormatter.new
+      logger
+    end
+
+    def self.logger
+      @logger ||= initialize_logger
+    end
+  end
+end

data/lib/upperkut/manager.rb

@@ -0,0 +1,50 @@
+require_relative 'core_ext'
+require_relative 'worker_thread'
+require_relative 'logging'
+require_relative 'worker'
+
+module Upperkut
+  class Manager
+    attr_accessor :worker
+    attr_reader :stopped, :logger, :concurrency
+
+    def initialize(opts = {})
+      self.worker = opts.fetch(:worker).constantize
+      @concurrency = opts.fetch(:concurrency, 1)
+      @logger = opts.fetch(:logger, Logging.logger)
+
+      @stopped = false
+      @threads = []
+    end
+
+    def run
+      @concurrency.times do
+        spawn_thread
+      end
+    end
+
+    def stop
+      @stopped = true
+      @threads.each(&:stop)
+    end
+
+    def kill
+      @threads.each(&:kill)
+    end
+
+    def notify_killed_processor(thread)
+      @threads.delete(thread)
+      spawn_thread unless @stopped
+    end
+
+    private
+
+    def spawn_thread
+      processor = Processor.new(worker, logger)
+
+      thread = WorkerThread.new(self, processor)
+      @threads << thread
+      thread.run
+    end
+  end
+end

data/lib/upperkut/middleware.rb

@@ -0,0 +1,35 @@
+module Upperkut
+  module Middleware
+    class Chain
+      attr_reader :items
+
+      def initialize
+        @items = []
+      end
+
+      def add(item)
+        return @items if @items.include?(item)
+
+        @items << item
+      end
+
+      def remove(item)
+        @items.delete(item)
+      end
+
+      def invoke(*args)
+        chain = @items.map(&:new)
+
+        traverse_chain = lambda do
+          if chain.empty?
+            yield
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+
+        traverse_chain.call
+      end
+    end
+  end
+end

data/lib/upperkut/middlewares/datadog.rb

@@ -0,0 +1,11 @@
+module Upperkut
+  module Middlewares
+    class Datadog
+      def call(worker, _items)
+        ::Datadog.tracer.trace(worker.name) do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/upperkut/middlewares/new_relic.rb

@@ -0,0 +1,23 @@
+module Upperkut
+  module Middlewares
+    class NewRelic
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+      def call(worker, _items)
+        perform_action_with_newrelic_trace(trace_args(worker)) do
+          yield
+        end
+      end
+
+      private
+
+      def trace_args(worker)
+        {
+          name: 'perform',
+          class_name: worker.name,
+          category: 'OtherTransaction/Upperkut'
+        }
+      end
+    end
+  end
+end

data/lib/upperkut/middlewares/rollbar.rb

@@ -0,0 +1,25 @@
+module Upperkut
+  module Middlewares
+    class Rollbar
+      def call(worker, items)
+        ::Rollbar.reset_notifier!
+        yield
+      rescue Exception => e
+        handle_exception(e, worker, items)
+        raise e
+      end
+
+      private
+
+      def handle_exception(e, worker, items)
+        scope = {
+          framework: "Upperkut #{::Upperkut::VERSION}",
+          request: { params: { items_size: items.size } },
+          context: worker.name
+        }
+
+        ::Rollbar.scope(scope).error(e, use_exception_level_filters: true)
+      end
+    end
+  end
+end

data/lib/upperkut/processor.rb

@@ -0,0 +1,64 @@
+require_relative 'logging'
+
+module Upperkut
+  class Processor
+    def initialize(worker, logger = Logging.logger)
+      @worker = worker
+      @strategy = worker.strategy
+      @worker_instance = worker.new
+      @logger = logger
+    end
+
+    def process
+      items = @worker.fetch_items.freeze
+      return unless items.any?
+
+      @worker.server_middlewares.invoke(@worker, items) do
+        @worker_instance.perform(items)
+      end
+
+      nacked_items, pending_ack_items = items.partition(&:nacked?)
+      @strategy.nack(nacked_items) if nacked_items.any?
+      @strategy.ack(pending_ack_items) if pending_ack_items.any?
+    rescue StandardError => error
+      @logger.error(
+        action: :handle_execution_error,
+        ex: error.to_s,
+        backtrace: error.backtrace.join("\n"),
+        item_size: Array(items).size
+      )
+
+      if items
+        if @worker_instance.respond_to?(:handle_error)
+          @worker_instance.handle_error(error, items)
+          return
+        end
+
+        @strategy.nack(items)
+      end
+
+      raise error
+    end
+
+    def blocking_process
+      sleeping_time = 0
+
+      loop do
+        break if @stopped
+
+        if @strategy.process?
+          sleeping_time = 0
+          process
+          next
+        end
+
+        sleeping_time += sleep(@worker.setup.polling_interval)
+        @logger.debug(sleeping_time: sleeping_time)
+      end
+    end
+
+    def stop
+      @stopped = true
+    end
+  end
+end

data/lib/upperkut/redis_pool.rb

@@ -0,0 +1,29 @@
+require 'connection_pool'
+require 'redis'
+
+module Upperkut
+  class RedisPool
+    DEFAULT_OPTIONS = {
+      pool_timeout: 1, # pool related option
+      size: 2, # pool related option
+      connect_timeout: 0.2,
+      read_timeout: 5.0,
+      write_timeout: 0.5
+    }.freeze
+
+    def initialize(options)
+      @options      = DEFAULT_OPTIONS.merge(url: ENV['REDIS_URL'])
+                                     .merge(options)
+
+      # Extract pool related options
+      @size         = @options.delete(:size)
+      @pool_timeout = @options.delete(:pool_timeout)
+    end
+
+    def create
+      ConnectionPool.new(timeout: @pool_timeout, size: @size) do
+        Redis.new(@options)
+      end
+    end
+  end
+end

data/lib/upperkut/strategies/base.rb

@@ -0,0 +1,56 @@
+module Upperkut
+  module Strategies
+    class Base
+      # Public: Ingests the event into strategy.
+      #
+      # items - The Array of items do be inserted.
+      #
+      # Returns true when success, raise when error.
+      def push_items(_items = [])
+        raise NotImplementedError
+      end
+
+      # Public: Retrieve events from Strategy.
+      #
+      # batch_size: # of items to be retrieved.
+      #
+      # Returns an Array containing events as hash.
+      def fetch_items(_batch_size)
+        raise NotImplementedError
+      end
+
+      # Public: Clear all data related to the strategy.
+      def clear
+        raise NotImplementedError
+      end
+
+      # Public: Confirms that items have been processed successfully.
+      #
+      # items - The Array of items do be confirmed.
+      def ack(_items)
+        raise NotImplementedError
+      end
+
+      # Public: Informs that items have been not processed successfully and therefore must be re-processed.
+      #
+      # items - The Array of items do be unacknowledged.
+      def nack(_items)
+        raise NotImplementedError
+      end
+
+      # Public: Tells when to execute the event processing,
+      # when this condition is met so the events are dispatched to
+      # the worker.
+      def process?
+        raise NotImplementedError
+      end
+
+      # Public: Consolidated strategy metrics.
+      #
+      # Returns hash containing metric name and values.
+      def metrics
+        raise NotImplementedError
+      end
+    end
+  end
+end