toiler 0.1.5 → 0.2.0

data/lib/toiler/logging.rb

@@ -1,42 +0,0 @@
-require 'time'
-require 'logger'
-
-module Toiler
-  module Logging
-    class Pretty < Logger::Formatter
-      # Provide a call() method that returns the formatted message.
-      def call(severity, time, _program_name, message)
-        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
-      end
-
-      def context
-        c = Thread.current[:toiler_context]
-        c ? " #{c}" : ''
-      end
-    end
-
-    module_function
-
-    def with_context(msg)
-      Thread.current[:toiler_context] = msg
-      yield
-    ensure
-      Thread.current[:toiler_context] = nil
-    end
-
-    def initialize_logger(log_target = STDOUT)
-      @logger = Logger.new(log_target)
-      @logger.level = Logger::INFO
-      @logger.formatter = Pretty.new
-      @logger
-    end
-
-    def logger
-      @logger || initialize_logger
-    end
-
-    def logger=(log)
-      @logger = (log ? log : Logger.new('/dev/null'))
-    end
-  end
-end

data/lib/toiler/manager.rb

@@ -1,71 +0,0 @@
-require 'toiler/fetcher'
-require 'toiler/processor'
-
-module Toiler
-  class Manager
-    include Celluloid
-    include Celluloid::Internals::Logger
-
-    finalizer :shutdown
-
-    def initialize
-      async.init
-    end
-
-    def init
-      debug 'Initializing manager...'
-      init_workers
-      awake_fetchers
-      debug 'Finished initializing manager...'
-    end
-
-    def awake_fetchers
-      queues.each do |q, _klass|
-        fetcher = Toiler.fetcher(q)
-        fetcher.processor_finished if fetcher && fetcher.alive? && free_processors(q) > 0
-      end
-    end
-
-    def queues
-      Toiler.worker_class_registry
-    end
-
-    def shutdown
-      debug 'Manager shutting down...'
-      instance_variables.each { |iv| remove_instance_variable iv }
-    end
-
-    def processor_finished(queue)
-      fetcher = Toiler.fetcher(queue)
-      fetcher.processor_finished if fetcher && fetcher.alive?
-    end
-
-    def init_workers
-      queues.each do |q, klass|
-        Toiler.worker_registry[q] = klass.new
-      end
-    end
-
-    def free_processors(queue)
-      pool = Toiler.processor_pool(queue)
-      pool && pool.alive? ? pool.idle_size : 0
-    end
-
-    def assign_messages(queue, messages)
-      debug "Manager assigning #{messages.count} for queue #{queue}"
-      processor_pool = Toiler.processor_pool(queue)
-      if batch? queue
-        processor_pool.async.process(queue, messages)
-      else
-        messages.each do |m|
-          processor_pool.async.process(queue, m)
-        end
-      end
-      debug "Manager finished assigning #{messages.count} for queue #{queue}"
-    end
-
-    def batch?(queue)
-      queues[queue].batch?
-    end
-  end
-end

data/lib/toiler/message.rb

@@ -1,60 +0,0 @@
-module Toiler
-  class Message
-    attr_accessor :client, :queue_url, :data
-
-    def initialize(client, queue_url, data)
-      @client = client
-      @queue_url = queue_url
-      @data = data
-    end
-
-    def delete
-      client.delete_message(
-        queue_url: queue_url,
-        receipt_handle: data.receipt_handle
-      )
-    end
-
-    def change_visibility(options)
-      client.change_message_visibility(
-        options.merge(queue_url: queue_url, receipt_handle: data.receipt_handle)
-      )
-    end
-
-    def visibility_timeout=(timeout)
-      client.change_message_visibility(
-        queue_url: queue_url,
-        receipt_handle: data.receipt_handle,
-        visibility_timeout: timeout
-      )
-    end
-
-    def message_id
-      data.message_id
-    end
-
-    def receipt_handle
-      data.receipt_handle
-    end
-
-    def md5_of_body
-      data.md5_of_body
-    end
-
-    def body
-      data.body
-    end
-
-    def attributes
-      data.attributes
-    end
-
-    def md5_of_message_attributes
-      data.md5_of_message_attributes
-    end
-
-    def message_attributes
-      data.message_attributes
-    end
-  end
-end

data/lib/toiler/processor.rb

@@ -1,86 +0,0 @@
-require 'json'
-require 'toiler/scheduler'
-
-module Toiler
-  class Processor
-    include Celluloid
-    include Celluloid::Internals::Logger
-
-    attr_accessor :queue
-
-    finalizer :shutdown
-
-    def initialize(queue)
-      debug "Initializing Processor for queue #{queue}"
-      @queue = queue
-      processor_finished
-      debug "Finished initializing Processor for queue #{queue}"
-    end
-
-    def shutdown
-      debug "Processor for queue #{queue} shutting down..."
-      ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
-      instance_variables.each { |iv| remove_instance_variable iv }
-    end
-
-    def process(queue, sqs_msg)
-      debug "Processor #{queue} begins processing..."
-      worker = Toiler.worker_registry[queue]
-      timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
-
-      body = get_body(worker.class, sqs_msg)
-      worker.perform(sqs_msg, body)
-      sqs_msg.delete if worker.class.auto_delete?
-    rescue StandardError => e
-      error "Processor #{queue} faild processing msg: #{e.message}\n#{e.backtrace.join("\n")}"
-    ensure
-      timer.cancel if timer
-      ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
-      processor_finished
-      debug "Processor #{queue} finishes processing..."
-    end
-
-    private
-
-    def processor_finished
-      Toiler.manager.async.processor_finished queue
-    end
-
-    def auto_visibility_timeout(queue, sqs_msg, worker_class)
-      return unless worker_class.auto_visibility_timeout?
-      queue_visibility_timeout = Toiler.fetcher(queue).queue.visibility_timeout
-      block = lambda do |msg, visibility_timeout, q|
-        debug "Processor #{q} updating visibility_timeout..."
-        msg.visibility_timeout = visibility_timeout
-      end
-
-      Toiler.scheduler(queue).custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, queue, block)
-    end
-
-    def get_body(worker_class, sqs_msg)
-      if sqs_msg.is_a? Array
-        sqs_msg.map { |m| parse_body(worker_class, m) }
-      else
-        parse_body(worker_class, sqs_msg)
-      end
-    end
-
-    def parse_body(worker_class, sqs_msg)
-      body_parser = worker_class.get_toiler_options[:parser]
-
-      case body_parser
-      when :json
-        JSON.parse(sqs_msg.body)
-      when Proc
-        body_parser.call(sqs_msg)
-      when :text, nil
-        sqs_msg.body
-      else
-        body_parser.load(sqs_msg.body) if body_parser.respond_to?(:load) # i.e. Oj.load(...) or MultiJson.load(...)
-      end
-    rescue => e
-      logger.error "Error parsing the message body: #{e.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}"
-      nil
-    end
-  end
-end

data/lib/toiler/queue.rb

@@ -1,53 +0,0 @@
-module Toiler
-  class Queue
-    attr_accessor :name, :client, :url
-
-    def initialize(name, client = nil)
-      @name   = name
-      @client = client || ::Aws::SQS::Client.new
-      @url    = client.get_queue_url(queue_name: name).queue_url
-    end
-
-    def visibility_timeout
-      client.get_queue_attributes(
-        queue_url: url,
-        attribute_names: ['VisibilityTimeout']
-      ).attributes['VisibilityTimeout'].to_i
-    end
-
-    def delete_messages(options)
-      client.delete_message_batch(options.merge(queue_url: url))
-    end
-
-    def send_message(options)
-      client.send_message(sanitize_message_body(options.merge(queue_url: url)))
-    end
-
-    def send_messages(options)
-      client.send_message_batch(sanitize_message_body(options.merge(queue_url: url)))
-    end
-
-    def receive_messages(options)
-      client.receive_message(options.merge(queue_url: url))
-        .messages
-        .map { |m| Message.new(client, url, m) }
-    end
-
-    private
-
-    def sanitize_message_body(options)
-      messages = options[:entries] || [options]
-
-      messages.each do |m|
-        body = m[:message_body]
-        if body.is_a?(Hash)
-          m[:message_body] = JSON.dump(body)
-        elsif !body.is_a? String
-          fail ArgumentError, "The message body must be a String and you passed a #{body.class}"
-        end
-      end
-
-      options
-    end
-  end
-end

data/lib/toiler/scheduler.rb

@@ -1,16 +0,0 @@
-module Toiler
-  class Scheduler
-    include Celluloid
-    include Celluloid::Internals::Logger
-
-    execute_block_on_receiver :custom_every
-
-    def custom_every(*args, block)
-      period = args[0]
-      block_args = args[1..-1]
-      every(period) do
-        block.call(*block_args)
-      end
-    end
-  end
-end

data/lib/toiler/supervisor.rb

@@ -1,66 +0,0 @@
-require 'toiler/manager'
-
-module Toiler
-  class Supervisor
-    attr_accessor :client, :config
-
-    def initialize
-      @client = ::Aws::SQS::Client.new
-      @config = Celluloid::Supervision::Configuration.new
-      define_manager
-      define_schedulers
-      define_processors
-      define_fetchers
-      @config.deploy
-    end
-
-    def queues
-      Toiler.worker_class_registry
-    end
-
-    def define_manager
-      @config.define type: Manager, as: :manager
-    end
-
-    def define_fetchers
-      queues.each do |queue, _klass|
-        @config.define type: Fetcher, as: "fetcher_#{queue}".to_sym, args: [queue, client]
-      end
-    end
-
-    def define_schedulers
-      queues.each do |queue, _klass|
-        @config.define type: Scheduler, as: "scheduler_#{queue}".to_sym, args: []
-      end
-    end
-
-    def define_processors
-      queues.each do |q, klass|
-        @config.define type: Celluloid::Supervision::Container::Pool,
-                       as: "processor_pool_#{q}".to_sym,
-                       args: [actors: Processor, size: klass.concurrency, args: [q]]
-      end
-    end
-
-    def stop
-      terminate_fetchers
-      terminate_processors
-      Toiler.manager.terminate if Toiler.manager && Toiler.manager.alive?
-      @config.shutdown
-    end
-
-    def terminate_fetchers
-      queues.each do |queue, _klass|
-        fetcher = Toiler.fetcher(queue)
-        fetcher.terminate if fetcher && fetcher.alive?
-      end
-    end
-
-    def terminate_processors
-      queues.each do |queue, _klass|
-        processor_pool = Toiler.processor_pool(queue)
-        processor_pool.terminate if processor_pool && processor_pool.alive?
-      end
-    end
-  end
-end