pika_que 0.1.0

data/lib/pika_que/reporters/log_reporter.rb

@@ -0,0 +1,13 @@
+module PikaQue
+  module Reporters
+    class LogReporter
+
+      def report(ex, clazz, msg)
+        PikaQue.logger.debug "error processing <#{msg}>"
+        PikaQue.logger.error "Exception #{ex.class} in #{clazz}: #{ex.message}" unless ex.nil?
+        PikaQue.logger.error ex.backtrace.join("\n") unless ex.nil? || ex.backtrace.nil?
+      end
+
+    end
+  end
+end

data/lib/pika_que/runner.rb

@@ -0,0 +1,24 @@
+module PikaQue
+  class Runner
+
+    def run
+      run_config = {}
+
+      # TODO anything to add to run_config?
+
+      @processors = []
+      PikaQue.config[:processors].each do |processor_hash|
+        _processor = PikaQue::Util.constantize(processor_hash[:processor]).new(processor_hash.merge(run_config))
+        _processor.start
+        @processors << _processor
+      end
+    end
+
+    # halt? pause?
+    def stop
+      @processors.each(&:stop)
+      PikaQue.connection.disconnect!
+    end
+
+  end
+end

data/lib/pika_que/subscriber.rb

@@ -0,0 +1,80 @@
+require 'pika_que/reporters'
+require 'pika_que/metrics'
+
+module PikaQue
+  class Subscriber
+    include Logging
+    include Reporters
+    include Metrics
+
+    attr_accessor :broker, :pool, :queue, :handler
+
+    def initialize(opts = {})
+      @opts = PikaQue.config.merge(opts)
+      @codec = @opts[:codec]
+      @broker = @opts[:broker] || PikaQue::Broker.new(nil, @opts).tap{ |b| b.start }
+      @pool = @opts[:worker_pool] || Concurrent::FixedThreadPool.new(@opts[:concurrency] || 1)
+    end
+
+    def setup_queue(queue_name, queue_opts)
+      @queue = broker.queue(queue_name, @opts[:queue_options].merge(queue_opts))
+    end
+
+    def setup_handler(handler_class, handler_opts)
+      @handler = broker.handler(handler_class, handler_opts)
+      # TODO use routing keys?
+      @handler.bind_queue(@queue, @queue.name)
+    end
+
+    def subscribe(worker)
+      @consumer = queue.subscribe(:block => false, :manual_ack => @opts[:ack], :arguments => worker.consumer_arguments) do | delivery_info, metadata, msg |
+        # TODO make idletime configurable on thread pool? default is 60.
+        pool.post do
+          res = nil
+          error = nil
+          begin
+            decoded_msg = @codec.decode(msg)
+            metrics.measure("work.#{self.class.name}.time") do
+              PikaQue.middleware.invoke(self, delivery_info, metadata, decoded_msg) do
+                res = worker.work(delivery_info, metadata, decoded_msg)
+              end
+            end
+            logger.debug "done processing #{res} <#{msg}>"
+          rescue => worker_err
+            res = :error
+            error = worker_err
+            notify_reporters(worker_err, worker.class, msg)
+          end
+
+          if @opts[:ack]
+            begin
+              handler.handle(res, broker.channel, delivery_info, metadata, msg, error)
+              metrics.increment("work.#{self.class.name}.handled.#{res}") 
+            rescue => handler_err
+              notify_reporters(handler_err, handler.class, msg)
+              metrics.increment("work.#{self.class.name}.handler.error") 
+            end
+          else
+            metrics.increment("work.#{self.class.name}.handled.noop") 
+          end
+          metrics.increment("work.#{self.class.name}.processed")
+        end
+      end
+    end
+
+    def unsubscribe
+      @consumer.cancel if @consumer
+      @consumer = nil
+    end
+
+    def teardown
+      unless @opts[:worker_pool]
+        @pool.shutdown
+        @pool.wait_for_termination 12
+      end
+      broker.cleanup
+      broker.stop
+    end
+
+  end
+end

data/lib/pika_que/util.rb

@@ -0,0 +1,17 @@
+module PikaQue
+  module Util
+    extend self
+    
+    def constantize(str)
+      return str if str.is_a? Class
+
+      names = str.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
+    end
+
+  end
+end

data/lib/pika_que/version.rb

@@ -0,0 +1,3 @@
+module PikaQue
+  VERSION = "0.1.0"
+end

data/lib/pika_que/worker.rb

@@ -0,0 +1,99 @@
+require 'pika_que/subscriber'
+
+module PikaQue
+  module Worker
+
+    def initialize(opts = {})
+      @subscriber = PikaQue::Subscriber.new(opts)
+    end
+
+    def prepare
+      @subscriber.setup_queue(self.class.queue_name, self.class.queue_opts)
+      @subscriber.setup_handler(self.class.handler_class, self.class.handler_opts)
+    end
+
+    def run
+      @subscriber.subscribe(self)
+    end
+
+    def start
+      prepare
+      run
+    end
+
+    def stop
+      @subscriber.unsubscribe
+      @subscriber.teardown
+    end
+
+    def work(delivery_info, metadata, msg)
+      perform(msg)
+    end
+
+    def ack!; :ack end
+    def reject!; :reject; end
+    def requeue!; :requeue; end
+
+    def logger
+      PikaQue.logger
+    end
+
+    def consumer_arguments
+      self.class.priority.nil? ? {} : { :'x-priority' => self.class.priority }
+    end
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      attr_reader :queue_name
+      attr_reader :queue_opts
+      attr_reader :handler_class
+      attr_reader :handler_opts
+      attr_reader :priority
+      attr_reader :config
+
+      def from_queue(q, opts={})
+        @queue_name = q.to_s
+        @priority = opts.delete(:priority)
+        @queue_opts = PikaQue.config[:queue_options].merge(opts)
+      end
+
+      def handle_with(handler, opts={})
+        @handler_class = handler
+        @handler_opts = PikaQue.config[:handler_options].merge(opts)
+      end
+
+      def enqueue(msg, opts={})
+        opts[:routing_key] ||= (queue_opts[:routing_key] if queue_opts)
+        opts[:to_queue] ||= queue_name
+        opts[:priority] ||= priority
+
+        publisher.publish(msg, opts)
+      end
+      alias_method :perform_async, :enqueue
+
+      def enqueue_at(msg, timestamp, opts={})
+        opts[:to_queue] ||= "#{PikaQue.config[:exchange]}-delay"
+        work_queue = opts.delete(:routing_key) || (queue_opts[:routing_key] if queue_opts) || queue_name
+        opts[:headers] = { work_at: timestamp, work_queue: work_queue }
+
+        publisher.publish(msg, opts)
+      end
+      alias_method :perform_at, :enqueue
+
+      def config(opts={})
+        @config = opts
+      end
+
+      private
+
+      def publisher
+        @publisher ||= PikaQue::Publisher.new(config || {})
+      end
+
+    end
+  
+  end
+end

data/pika_que.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pika_que/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pika_que"
+  spec.version       = PikaQue::VERSION
+  spec.authors       = ["Dong Wook Koo"]
+  spec.email         = ["dwkoogt@gmail.com"]
+
+  spec.summary       = %q{Ruby background processor for RabbitMQ.}
+  spec.description   = %q{Ruby background processor for RabbitMQ.}
+  spec.homepage      = "https://github.com/dwkoogt/pika_que"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  # end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'bunny', '~> 2.6'
+  spec.add_dependency 'concurrent-ruby', '~> 1.0'
+  spec.add_dependency 'json', '~> 1.8'
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification
+name: pika_que
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dong Wook Koo
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-03-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Ruby background processor for RabbitMQ.
+email:
+- dwkoogt@gmail.com
+executables:
+- pika_que
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/demo.rb
+- examples/demo_delay.rb
+- examples/demo_oneoff.rb
+- examples/demo_priority.rb
+- examples/demo_reporter.rb
+- examples/demo_retry.rb
+- examples/demo_worker.rb
+- examples/dev_worker.rb
+- exe/pika_que
+- lib/active_job/queue_adapters/pika_que_adapter.rb
+- lib/pika_que.rb
+- lib/pika_que/broker.rb
+- lib/pika_que/cli.rb
+- lib/pika_que/codecs/json.rb
+- lib/pika_que/codecs/noop.rb
+- lib/pika_que/codecs/rails.rb
+- lib/pika_que/configuration.rb
+- lib/pika_que/connection.rb
+- lib/pika_que/delay_worker.rb
+- lib/pika_que/errors.rb
+- lib/pika_que/handlers/default_handler.rb
+- lib/pika_que/handlers/delay_handler.rb
+- lib/pika_que/handlers/error_handler.rb
+- lib/pika_que/handlers/retry_handler.rb
+- lib/pika_que/launcher.rb
+- lib/pika_que/logging.rb
+- lib/pika_que/metrics.rb
+- lib/pika_que/metrics/log_metric.rb
+- lib/pika_que/metrics/null_metric.rb
+- lib/pika_que/middleware/active_record.rb
+- lib/pika_que/middleware/chain.rb
+- lib/pika_que/processor.rb
+- lib/pika_que/publisher.rb
+- lib/pika_que/rails.rb
+- lib/pika_que/reporters.rb
+- lib/pika_que/reporters/log_reporter.rb
+- lib/pika_que/runner.rb
+- lib/pika_que/subscriber.rb
+- lib/pika_que/util.rb
+- lib/pika_que/version.rb
+- lib/pika_que/worker.rb
+- pika_que.gemspec
+homepage: https://github.com/dwkoogt/pika_que
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.2
+signing_key: 
+specification_version: 4
+summary: Ruby background processor for RabbitMQ.
+test_files: []