pika_que 0.1.0

data/exe/pika_que

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require "pika_que"
+require "pika_que/cli"
+
+cli = PikaQue::CLI.new
+cli.parse
+cli.run

data/lib/active_job/queue_adapters/pika_que_adapter.rb

@@ -0,0 +1,42 @@
+require 'pika_que'
+require 'pika_que/codecs/rails'
+require 'thread'
+
+module ActiveJob
+  module QueueAdapters
+    # == PikaQue adapter for Active Job
+    #
+    # PikaQue is a RabbitMQ background processing framework for Ruby.
+    #
+    # Read more about PikaQue {here}[https://github.com/dwkoogt/pika_que].
+    #
+    # To use PikaQue set the queue_adapter config to +:pika_que+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :pika_que
+    #
+    class PikaQueAdapter
+      @monitor = Monitor.new
+
+      class << self
+        def enqueue(job) #:nodoc:
+          @monitor.synchronize do
+            JobWrapper.enqueue job.serialize, to_queue: job.queue_name
+          end
+        end
+
+        def enqueue_at(job, timestamp) #:nodoc:
+          @monitor.synchronize do
+            JobWrapper.enqueue_at job.serialize, timestamp, routing_key: job.queue_name
+          end
+        end
+      end
+
+      class JobWrapper #:nodoc:
+        extend PikaQue::Worker::ClassMethods
+        config codec: PikaQue::Codecs::RAILS
+      end
+    end
+
+    autoload :PikaQueAdapter
+  end
+end

data/lib/pika_que.rb

@@ -0,0 +1,44 @@
+require 'bunny'
+require 'concurrent/executors'
+
+require 'pika_que/configuration'
+require 'pika_que/errors'
+require 'pika_que/logging'
+require 'pika_que/version'
+
+require 'pika_que/connection'
+require 'pika_que/publisher'
+require 'pika_que/reporters/log_reporter'
+require 'pika_que/middleware/chain'
+require 'pika_que/worker'
+
+module PikaQue
+
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  def self.logger
+    PikaQue::Logging.logger
+  end
+
+  def self.logger=(logger)
+    PikaQue::Logging.logger = logger
+  end
+
+  def self.connection
+    @connection ||= Connection.create
+  end
+
+  def self.middleware
+    @chain ||= Middleware::Chain.new
+    yield @chain if block_given?
+    @chain
+  end
+
+  def self.reporters
+    config[:reporters] << PikaQue::Reporters::LogReporter.new if config[:reporters].empty?
+    config[:reporters]
+  end
+
+end

data/lib/pika_que/broker.rb

@@ -0,0 +1,88 @@
+require 'pika_que/util'
+require 'pika_que/handlers/default_handler'
+
+module PikaQue
+  class Broker
+
+    def initialize(processor = nil, opts = {})
+      @opts = PikaQue.config.merge(opts)
+      @processor = processor
+      @handlers = {}
+    end
+
+    def start
+      @connection ||= @opts[:connection_options] ? PikaQue::Connection.create(@opts[:connection_options]) : PikaQue.connection
+      @connection.ensure_connection
+    end
+
+    def stop
+      @connection.disconnect! if local_connection?
+    end
+
+    def local_connection?
+      @opts[:connection_options] || @processor.nil?
+    end
+
+    def queue(queue_name, queue_opts = {})
+      begin
+        queue = channel.queue(queue_name, queue_opts)
+        routing_key = queue_opts[:routing_key] || queue_name
+        routing_keys = [routing_key, *queue_opts[:routing_keys]]
+
+        routing_keys.each do |key|
+          queue.bind(exchange, routing_key: key)
+        end
+        queue
+      rescue => e
+        PikaQue.logger.fatal e.message
+        raise SetupError.new e.message
+      end
+    end
+
+    def handler(handler_class, handler_opts = {})
+      if handler_class
+        h_key = "#{handler_class}-#{handler_opts.hash}"
+        _handler = @handlers[h_key]
+        unless _handler
+          _handler = handler_class.new(handler_opts.merge({ connection: @connection }))
+          @handlers[h_key] = _handler
+        end
+        _handler
+      else
+        default_handler
+      end
+    end
+
+    def default_handler
+      @default_handler ||= @opts[:handler_class] ? PikaQue::Util.constantize(@opts[:handler_class]).new(@opts[:handler_options].merge({ connection: @connection })) : PikaQue::Handlers::DefaultHandler.new
+    end
+
+    def exchange
+      @exchange ||= channel.exchange(@opts[:exchange], @opts[:exchange_options])
+    end
+
+    def channel
+      @channel ||= init_channel
+    end
+
+    def init_channel
+      @connection.create_channel(nil, @opts[:channel_options][:consumer_pool_size]).tap do |ch|
+        ch.prefetch(@opts[:channel_options][:prefetch])
+      end
+    end
+
+    def cleanup(force = false)
+      if (@processor && force) || !@processor
+        @channel.close unless @channel.closed?
+        @channel = nil
+        @exchange = nil
+        if @default_handler
+          @default_handler.close
+          @default_handler = nil
+        end
+        @handlers.values.each(&:close)
+      end
+    end
+
+  end
+end

data/lib/pika_que/cli.rb

@@ -0,0 +1,180 @@
+require 'optparse'
+
+require 'pika_que/connection'
+require 'pika_que/launcher'
+require 'pika_que/runner'
+
+module PikaQue
+  class CLI
+
+    attr_accessor :environment
+
+    def parse(args = ARGV)
+      opts = parse_options(args)
+      config.merge!(opts)
+      init_logger
+      daemonize
+      write_pid
+    end
+
+    def run
+
+      load_app
+
+      PikaQue.middleware
+
+      runner = Runner.new
+
+      begin
+
+        Launcher.launch(runner) do
+          runner.run
+        end
+
+        exit 0
+      rescue Interrupt, SetupError => e
+        PikaQue.logger.info "Shutting down: #{e.class.name} received"
+        runner.stop
+        exit 1
+      end
+    end
+
+    def config
+      PikaQue.config
+    end
+
+    def init_logger
+      PikaQue::Logging.init_logger(config[:logfile]) if config[:logfile]
+      PikaQue.logger.level = ::Logger::WARN if config[:quite]
+      PikaQue.logger.level = ::Logger::DEBUG if config[:verbose]
+    end
+
+    def daemonize
+      return unless config[:daemon]
+
+      files_to_reopen = []
+      ObjectSpace.each_object(File) do |file|
+        files_to_reopen << file unless file.closed?
+      end
+
+      ::Process.daemon(true, true)
+
+      files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, "a+"
+          file.sync = true
+        rescue ::Exception
+        end
+      end
+
+      [$stdout, $stderr].each do |io|
+        File.open(config[:logfile], 'ab') do |f|
+          io.reopen(f)
+        end
+        io.sync = true
+      end
+      $stdin.reopen(File::NULL)
+
+      init_logger
+    end
+
+    def write_pid
+      if path = config[:pidfile]
+        pidfile = File.expand_path(path)
+        File.open(pidfile, 'w') do |f|
+          f.puts ::Process.pid
+        end
+      end
+    end
+
+    def load_app
+      if File.directory?(config[:require])
+        rails_path = File.expand_path(File.join(config[:require], 'config', 'environment.rb'))
+        if File.exist?(rails_path)
+          ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
+          PikaQue.logger.info "found rails project (#{config[:require]}), booting app in #{ENV['RACK_ENV']} environment"
+          require 'rails'
+          require 'pika_que/rails'
+          require rails_path
+          ::Rails.application.eager_load!
+        end
+      else
+        require(File.expand_path(config[:require])) || raise(ArgumentError, 'require returned false')
+      end
+
+      if config[:delay]
+        config.add_processor(config.delete(:delay_options))
+      else
+        config.delete(:delay_options)
+      end
+
+      if config[:workers]
+        config.add_processor({ workers: config.delete(:workers) })
+      end
+
+    end
+
+    def parse_options(args)
+      opts = {}
+
+      @parser = OptionParser.new do |o|
+        o.banner = 'usage: pika_que [options]'
+
+        o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
+          opts[:concurrency] = Integer(arg)
+        end
+
+        o.on '-d', '--daemon', "Daemonize process" do |arg|
+          opts[:daemon] = arg
+        end
+
+        o.on '-e', '--environment ENV', "Application environment" do |arg|
+          opts[:environment] = arg
+        end
+
+        o.on '-q', '--quite', "Print quite output" do |arg|
+          opts[:quite] = arg
+        end
+
+        o.on '-v', '--verbose', "Print verbose output" do |arg|
+          opts[:verbose] = arg
+        end
+
+        o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
+          opts[:require] = arg
+        end
+
+        o.on '-w', '--worker WORKER(S)', "comma separated list of workers" do |arg|
+          opts[:workers] = arg.split(",")
+        end
+
+        o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
+          opts[:logfile] = arg
+        end
+
+        o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+          opts[:pidfile] = arg
+        end
+
+        o.on '-V', '--version', "Print version and exit" do
+          puts "PikaQue #{PikaQue::VERSION}"
+          exit 0
+        end
+
+        o.on_tail '-h', '--help', 'Show this message and exit' do
+          puts o
+          exit 0
+        end
+      end
+
+      @parser.parse!(args)
+
+      @environment = opts[:environment] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+
+      opts[:logfile] ||= opts[:daemon] ? 'pika_que.log' : STDOUT
+
+      opts
+    end
+    
+  end
+end

data/lib/pika_que/codecs/json.rb

@@ -0,0 +1,22 @@
+require 'json'
+
+module PikaQue
+  module Codecs
+    module JSON
+      extend self
+
+      def encode(payload)
+        ::JSON.generate(payload, quirks_mode: true)
+      end
+
+      def decode(payload)
+        ::JSON.parse(payload, quirks_mode: true)
+      end
+
+      def content_type
+        'application/json'
+      end
+
+    end
+  end
+end

data/lib/pika_que/codecs/noop.rb

@@ -0,0 +1,20 @@
+module PikaQue
+  module Codecs
+    module NOOP
+      extend self
+
+      def encode(payload)
+        payload
+      end
+
+      def decode(payload)
+        payload
+      end
+
+      def content_type
+        nil
+      end
+
+    end
+  end
+end

data/lib/pika_que/codecs/rails.rb

@@ -0,0 +1,22 @@
+require 'active_support/json'
+
+module PikaQue
+  module Codecs
+    module RAILS
+      extend self
+
+      def encode(payload)
+        ::ActiveSupport::JSON.encode(payload)
+      end
+
+      def decode(payload)
+        ::ActiveSupport::JSON.decode(payload)
+      end
+
+      def content_type
+        'application/json'
+      end
+
+    end
+  end
+end

data/lib/pika_que/configuration.rb

@@ -0,0 +1,110 @@
+require 'forwardable'
+
+require 'pika_que/codecs/json'
+require 'pika_que/codecs/noop'
+
+require 'pika_que/processor'
+require 'pika_que/handlers/default_handler'
+
+require 'pika_que/delay_worker'
+require 'pika_que/handlers/delay_handler'
+
+module PikaQue
+  class Configuration
+    extend Forwardable
+
+    def_delegators :@config, :to_hash, :[], :[]=, :==, :fetch, :delete, :has_key?
+
+    EXCHANGE_OPTION_DEFAULTS = {
+      :type               => :direct,
+      :durable            => true,
+      :auto_delete        => false,
+      :arguments => {} # Passed as :arguments to Bunny::Channel#exchange
+    }.freeze
+
+    QUEUE_OPTION_DEFAULTS = {
+      :durable            => true,
+      :auto_delete        => false,
+      :exclusive          => false,
+      :arguments => {}
+    }.freeze
+
+    CHANNEL_OPTION_DEFAULTS = {
+      :consumer_pool_size => 1,
+      :prefetch           => 10
+    }.freeze
+
+    DELAY_PROCESSOR_DEFAULTS = {
+      :workers            => [PikaQue::DelayWorker],
+      :handler_class      => PikaQue::Handlers::DelayHandler,
+      :concurrency        => 1
+    }.freeze
+  
+    DEFAULT_CONFIG = {
+      :exchange           => 'pika-que',
+      :heartbeat          => 30,
+      :channel_options    => CHANNEL_OPTION_DEFAULTS,
+      :exchange_options   => EXCHANGE_OPTION_DEFAULTS,
+      :queue_options      => QUEUE_OPTION_DEFAULTS,
+      :concurrency        => 1,
+      :ack                => true,
+      :handler_class      => PikaQue::Handlers::DefaultHandler,
+      :handler_options    => {},
+      :codec              => PikaQue::Codecs::JSON,
+      :processors         => [],
+      :reporters          => [],
+      :metrics            => nil,
+      :delay              => true,
+      :delay_options      => DELAY_PROCESSOR_DEFAULTS,
+      :pidfile            => nil,
+      :require            => '.'
+    }.freeze
+
+    # processor example
+    # @processor            Processor class
+    # @workers              array of worker classes
+    # @connection           connection params if using separate connection
+    # {
+    #   :processor          => Processor,
+    #   :connection_options => {},
+    #   :workers            => [],
+    #   :concurrency        => 1,
+    #   :ack                => true,
+    #   :handler_class      => nil,
+    #   :handler_options    => nil,
+    #   :codec              => PikaQue::Codecs::JSON
+    # }
+
+    def initialize
+      @config = DEFAULT_CONFIG.dup
+    end
+
+    def merge!(other = {})
+      @config = deep_merge(@config, other)
+    end
+
+    def merge(other = {})
+      instance = self.class.new
+      instance.merge! to_hash
+      instance.merge! other
+      instance
+    end
+
+    def deep_merge(first, second)
+      merger = proc { |_, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+      first.merge(second, &merger)
+    end
+
+    def processor(opts = {})
+      {
+        :processor        => PikaQue::Processor,
+        :workers          => []
+      }.merge(opts)
+    end
+
+    def add_processor(opts = {})
+      @config[:processors] << processor(opts)
+    end
+
+  end
+end