pika_que 0.1.0

data/lib/pika_que/launcher.rb

@@ -0,0 +1,92 @@
+module PikaQue
+  class Launcher
+    include Logging
+
+    def initialize(runnable)
+      @runnable = runnable
+    end
+
+    def self.launch(runnable)
+      if block_given?
+        new(runnable).launch { yield }
+      else
+        new(runnable).launch
+      end
+    end
+
+    def launch
+      @sig_read, @sig_write = IO.pipe
+
+      register_signals
+
+      if block_given?
+        yield
+      else
+        runnable.run
+      end
+
+      while readable_io = wait_for_signal
+        signal = readable_io.first[0].gets.strip
+        handle_signal(signal)
+      end
+      
+    end
+
+    private
+
+    attr_accessor :runnable, :sig_read, :sig_write
+
+    def register_signals
+      signals = %w(INT TERM TTIN TSTP)
+      if !defined?(::JRUBY_VERSION)
+        signals += %w(USR1 USR2)
+      end
+
+      signals.each do |sig|
+        trap sig do
+          sig_write.puts(sig)
+        end if Signal.list.keys.include?(sig)
+      end
+    end
+
+    def wait_for_signal
+      IO.select([sig_read])
+    end
+
+    def handle_signal(signal)
+      case signal
+      when 'INT'
+        logger.info "Received INT"
+        raise Interrupt
+      when 'TERM'
+        logger.info "Received TERM"
+        raise Interrupt
+      when 'TTIN'
+        logger.info "Received TTIN"
+        log_thread_backtraces
+      when 'TSTP'
+        logger.info "Received TSTP"
+        runnable.stop
+      when 'USR1'
+        logger.info "Received USR1"
+        runnable.stop
+      when 'USR2'
+        logger.info "Received USR2"
+        # TODO ?
+      end
+    end
+
+    def log_thread_backtraces
+      Thread.list.each do |thread|
+        logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+        if thread.backtrace
+            logger.warn thread.backtrace.join("\n")
+          else
+            logger.warn "<no backtrace available>"
+          end
+      end
+    end
+
+
+  end
+end

data/lib/pika_que/logging.rb

@@ -0,0 +1,33 @@
+require 'logger'
+require 'time'
+
+module PikaQue
+  module Logging
+
+    class PikaQueFormatter < Logger::Formatter
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} T-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n"
+      end
+    end
+
+    def self.init_logger(stream = STDOUT)
+      @logger = Logger.new(stream, 5, 1048576).tap do |l|
+        l.level = Logger::INFO
+        l.formatter = PikaQueFormatter.new
+      end
+    end
+
+    def self.logger
+      @logger || init_logger
+    end
+
+    def self.logger=(logger)
+      @logger = logger ? logger : Logger.new(File::NULL)
+    end
+
+    def logger
+      PikaQue::Logging.logger
+    end
+
+  end
+end

data/lib/pika_que/metrics.rb

@@ -0,0 +1,26 @@
+require 'pika_que/metrics/log_metric'
+require 'pika_que/metrics/null_metric'
+
+module PikaQue
+  module Metrics
+
+    def self.metrics
+      @metrics || init_metrics
+    end
+
+    def self.init_metrics
+      if PikaQue.config[:metrics]
+        @metrics = PikaQue.config[:metrics].new
+      elsif PikaQue.config[:quite]
+        @metrics = PikaQue::Metrics::NullMetric.new
+      else
+        @metrics = PikaQue::Metrics::LogMetric.new
+      end
+    end
+
+    def metrics
+      PikaQue::Metrics.metrics
+    end
+
+  end
+end

data/lib/pika_que/metrics/log_metric.rb

@@ -0,0 +1,23 @@
+require 'concurrent/map'
+
+module PikaQue
+  module Metrics
+    class LogMetric
+
+      COUNTERS = Concurrent::Map.new
+
+      def increment(metric, delta = 1)
+        COUNTERS[metric] = 0 unless COUNTERS[metric]
+        COUNTERS[metric] = COUNTERS[metric] + delta
+        PikaQue.logger.info("COUNT: #{metric} #{COUNTERS[metric]}")
+      end
+
+      def measure(metric, &block)
+        start = Time.now
+        block.call
+        PikaQue.logger.info("TIME: #{metric} #{Time.now - start}")
+      end
+
+    end
+  end
+end

data/lib/pika_que/metrics/null_metric.rb

@@ -0,0 +1,14 @@
+module PikaQue
+  module Metrics
+    class NullMetric
+
+      def increment(metric, delta = 1)
+      end
+
+      def measure(metric, &block)
+        block.call
+      end
+
+    end
+  end
+end

data/lib/pika_que/middleware/active_record.rb

@@ -0,0 +1,13 @@
+module PikaQue
+  module Middleware
+    class ActiveRecord
+
+      def call(*args)
+        yield
+      ensure
+        ::ActiveRecord::Base.clear_active_connections!
+      end
+      
+    end
+  end
+end

data/lib/pika_que/middleware/chain.rb

@@ -0,0 +1,90 @@
+#
+# lifted from Sidekiq
+# https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/middleware/chain.rb
+#
+module PikaQue
+  module Middleware
+    class Chain
+      include Enumerable
+      attr_reader :entries
+
+      def initialize_copy(copy)
+        copy.instance_variable_set(:@entries, entries.dup)
+      end
+
+      def each(&block)
+        entries.each(&block)
+      end
+
+      def initialize
+        @entries = []
+        yield self if block_given?
+      end
+
+      def remove(klass)
+        entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      def add(klass, *args)
+        remove(klass) if exists?(klass)
+        entries << Entry.new(klass, *args)
+      end
+
+      def prepend(klass, *args)
+        remove(klass) if exists?(klass)
+        entries.insert(0, Entry.new(klass, *args))
+      end
+
+      def insert_before(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || 0
+        entries.insert(i, new_entry)
+      end
+
+      def insert_after(oldklass, newklass, *args)
+        i = entries.index { |entry| entry.klass == newklass }
+        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
+        entries.insert(i+1, new_entry)
+      end
+
+      def exists?(klass)
+        any? { |entry| entry.klass == klass }
+      end
+
+      def retrieve
+        map(&:make_new)
+      end
+
+      def clear
+        entries.clear
+      end
+
+      def invoke(*args)
+        chain = retrieve.dup
+        traverse_chain = lambda do
+          if chain.empty?
+            yield
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+        traverse_chain.call
+      end
+    end
+
+    class Entry
+      attr_reader :klass
+
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/pika_que/processor.rb

@@ -0,0 +1,45 @@
+require 'pika_que/broker'
+require 'pika_que/logging'
+require 'pika_que/util'
+
+module PikaQue
+  class Processor
+    include Logging
+
+    def initialize(opts = {})
+      @opts = PikaQue.config.merge(opts)
+      @broker = PikaQue::Broker.new(self, @opts).tap{ |b| b.start }
+      @pool = Concurrent::FixedThreadPool.new(@opts[:concurrency] || 1)
+      proc_config = @opts.merge({ broker: @broker, worker_pool: @pool })
+      @workers = @opts.fetch(:workers, []).map{ |w| PikaQue::Util.constantize(w).new(proc_config) }
+      @thread = nil
+    end
+
+    def setup
+      @workers.each(&:prepare)
+    end
+
+    def process
+      @workers.each(&:run)
+    end
+
+    def start
+      @thread = Thread.new do
+        Thread.current['label'] = 'processor'
+        setup
+        process
+      end.abort_on_exception = true
+    end
+
+    def stop
+      @workers.each(&:stop)
+
+      @pool.shutdown
+      @pool.wait_for_termination 12
+
+      @broker.cleanup(true)
+      @broker.stop
+    end
+
+  end
+end

data/lib/pika_que/publisher.rb

@@ -0,0 +1,23 @@
+module PikaQue
+  class Publisher
+
+    def initialize(opts = {})
+      @opts = PikaQue.config.merge(opts)
+      @connection = @opts[:connection] || PikaQue.connection
+      @channel = @connection.create_channel
+      @exchange = @channel.exchange(@opts[:exchange], @opts[:exchange_options])
+    end
+
+    def publish(msg, options = {})
+      to_queue = options.delete(:to_queue)
+      codec = @opts[:codec]
+      options[:routing_key] ||= to_queue
+      options[:content_type] ||= codec.content_type
+      msg = codec.encode(msg)
+      
+      PikaQue.logger.info {"publishing <#{msg}> to [#{options[:routing_key]}]"}
+      @exchange.publish(msg, options)
+    end
+
+  end
+end

data/lib/pika_que/rails.rb

@@ -0,0 +1,62 @@
+require 'pika_que/worker'
+require 'pika_que/codecs/rails'
+
+module PikaQue
+  class Rails < ::Rails::Engine
+
+    config.before_configuration do
+      if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
+        PikaQue.middleware do |chain|
+          require 'pika_que/middleware/active_record'
+          chain.add PikaQue::Middleware::ActiveRecord
+        end
+      end
+    end
+
+    config.before_initialize do
+      require 'active_job/queue_adapters/pika_que_adapter'
+    end
+
+    config.after_initialize do
+      config_file = Rails.root.join('config').join('pika_que.yml')
+      if File.exist? config_file
+        PIKA_QUE_CONFIG = YAML.load_file(config_file)
+      else
+        PIKA_QUE_CONFIG = { "processors" => [{ "workers" => [{ "queue" => ActiveJob::Base.queue_name }, { "queue" => ActionMailer::DeliveryJob.queue_name}] }] }
+      end
+
+      workers_dir = Rails.root.join('app').join('workers')
+      if Dir.exist? workers_dir
+        worker_files = Dir.glob(workers_dir.join('*.rb'))
+      else
+        worker_files = []
+      end
+
+      # TODO options, etc
+
+      PIKA_QUE_CONFIG['processors'].each do |processor|
+        workers = []
+        processor['workers'].each do |worker|
+          queue = worker['queue']
+          worker_name = worker['worker'] || "#{queue.classify}Worker"
+          Object.const_set(worker_name, Class.new do
+              include PikaQue::Worker
+              from_queue queue
+              config codec: PikaQue::Codecs::RAILS
+
+              def perform(msg)
+                ActiveJob::Base.execute msg
+                ack!
+              end
+            end
+          ) unless worker_files.detect{ |w| w =~ /#{worker_name.snakecase}/ }
+          workers << worker_name
+        end
+        proc_args = { workers: workers }
+        proc_args[:processor] = processor['processor'] if processor['processor']
+        PikaQue.config.add_processor(proc_args)
+      end
+    end
+
+  end
+end

data/lib/pika_que/reporters.rb

@@ -0,0 +1,18 @@
+module PikaQue
+  module Reporters
+
+    def notify_reporters(ex, clazz, msg)
+      PikaQue.reporters.each do |reporter|
+        begin
+          reporter.report(ex, clazz, msg)
+        rescue => e
+          PikaQue.logger.error "error reporting by #{reporter.class}"
+          PikaQue.logger.error e
+          PikaQue.logger.error e.backtrace.join("\n") unless e.backtrace.nil?
+        end
+        
+      end
+    end
+    
+  end
+end