vx-common-amqp 0.2.6

data/lib/vx/common/amqp/consumer/publish.rb

@@ -0,0 +1,32 @@
+module Vx
+  module Common
+    module AMQP
+      module Consumer::Publish
+
+        def publish(message, options = nil)
+          session.open
+
+          options ||= {}
+          options[:routing_key]  = routing_key if routing_key && !options.key?(:routing_key)
+          options[:headers]      = headers     if headers && !options.key?(:headers)
+          options[:content_type] ||= content_type || config.content_type
+
+          x = declare_exchange
+
+          run_callbacks(:publish, message: message, exchange: x, name: consumer_id) do
+            m = serialize_message message, options[:content_type]
+            x.publish m, options
+          end
+
+          debug "published #{message.inspect} to #{x.name}"
+          self
+        end
+
+        def serialize_message(message, content_type)
+          Common::AMQP::Formatter.pack(content_type, message)
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/consumer/subscribe.rb

@@ -0,0 +1,67 @@
+module Vx
+  module Common
+    module AMQP
+      module Consumer::Subscribe
+
+        def subscribe
+          session.open
+
+          session.with_channel do
+            x = declare_exchange
+            q = declare_queue
+
+            run_callbacks(:subscribe, exchange: x, queue: q, name: consumer_id) do
+              debug "subscribing to #{q.name}:#{x.name} using #{bind_options.inspect}"
+              q.bind(x, bind_options)
+              debug "successfuly subscribed to #{q.name}:#{x.name}"
+
+              subscription_loop q
+            end
+
+            debug "shutdown"
+          end
+        end
+
+        private
+
+          def subscription_loop(q)
+            loop do
+              break if shutdown?
+
+              delivery_info, properties, payload = q.pop(ack: ack)
+
+              if payload
+                result = nil
+
+                debug "recieve ##{delivery_info.delivery_tag.to_i} #{payload.inspect}"
+                result = run_instance delivery_info, properties, payload
+                debug "done ##{delivery_info.delivery_tag.to_i}"
+
+                break if result == :shutdown
+              else
+                sleep config.pool_timeout
+              end
+            end
+          end
+
+          def run_instance(delivery_info, properties, payload)
+            payload = deserialize_message properties, payload
+
+            run_callbacks :recieve, payload: payload, name: consumer_id do
+              new.tap do |inst|
+                inst.properties    = properties
+                inst.delivery_info = delivery_info
+              end.perform payload
+            end
+          end
+
+          def deserialize_message(properties, payload)
+            Common::AMQP::Formatter.unpack properties[:content_type],
+                                           model,
+                                           payload
+          end
+
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/consumer.rb

@@ -0,0 +1,70 @@
+require 'ostruct'
+
+module Vx
+  module Common
+    module AMQP
+      module Consumer
+
+        autoload :Configuration, File.expand_path("../consumer/configuration", __FILE__)
+        autoload :Publish,       File.expand_path("../consumer/publish",       __FILE__)
+        autoload :Subscribe,     File.expand_path("../consumer/subscribe",     __FILE__)
+        autoload :Sleep,         File.expand_path("../consumer/sleep",         __FILE__)
+        autoload :Ack,           File.expand_path("../consumer/ack",           __FILE__)
+
+        include Common::AMQP::Consumer::Ack
+        include Common::AMQP::Logger
+
+        attr_accessor :delivery_info
+        attr_accessor :properties
+        attr_accessor :channel
+
+        @@classes = []
+
+        def self.included(base)
+          base.extend ClassMethods
+          @@classes << base.to_s
+        end
+
+        def self.classes
+          @@classes
+        end
+
+        module ClassMethods
+
+          include Common::AMQP::Consumer::Configuration
+          include Common::AMQP::Consumer::Publish
+          include Common::AMQP::Consumer::Subscribe
+          include Common::AMQP::Logger
+          include Common::AMQP::Callbacks
+
+          def shutdown?
+            Common::AMQP.shutdown?
+          end
+
+          def shutdown
+            Common::AMQP.shutdown
+          end
+
+          def session
+            Common::AMQP.session
+          end
+
+          def config
+            Common::AMQP.config
+          end
+
+          private
+
+            def declare_exchange
+              session.declare_exchange(exchange_name, exchange_options)
+            end
+
+            def declare_queue
+              session.declare_queue(queue_name, queue_options)
+            end
+
+        end
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/formatter.rb

@@ -0,0 +1,105 @@
+require 'json'
+require 'stringio'
+
+module Vx
+  module Common
+    module AMQP
+
+      class Formatter
+
+        @@formats = {}
+
+        class Format
+
+          attr_reader :content_type
+
+          def initialize(content_type)
+            @content_type = content_type
+          end
+
+          def pack(&block)
+            @pack = block if block_given?
+            @pack
+          end
+
+          def unpack(&block)
+            @unpack = block if block_given?
+            @unpack
+          end
+
+        end
+
+        class << self
+
+          def formats
+            @@formats
+          end
+
+          def define(content_type, &block)
+            fmt = Format.new content_type
+            fmt.instance_eval(&block)
+            formats.merge! content_type => fmt
+          end
+
+          def lookup(content_type)
+            formats[content_type]
+          end
+
+          def pack(content_type, body)
+            if fmt = lookup(content_type)
+              fmt.pack.call(body)
+            end
+          end
+
+          def unpack(content_type, model, body)
+            if fmt = lookup(content_type)
+              fmt.unpack.call(body, model)
+            end
+          end
+
+        end
+
+        define 'text/plain' do
+
+          pack do |body|
+            body.to_s
+          end
+
+          unpack do |body, _|
+            body
+          end
+        end
+
+        define 'application/json' do
+
+          pack do |body|
+            body.to_json
+          end
+
+          unpack do |payload, model|
+            if model && model.respond_to?(:from_json)
+              model.from_json payload
+            else
+              JSON.parse(payload)
+            end
+          end
+        end
+
+        define 'application/x-protobuf' do
+
+          pack do |object|
+            object.encode.to_s
+          end
+
+          unpack do |payload, model|
+            raise ModelDoesNotExists unless model
+            model.decode payload
+          end
+
+          class ModelDoesNotExists < Exception ; end
+        end
+      end
+
+    end
+  end
+end

data/lib/vx/common/amqp/mixins/callbacks.rb

@@ -0,0 +1,35 @@
+module Vx
+  module Common
+    module AMQP
+      module Callbacks
+
+        def run_callbacks(name, *args)
+          before = "before_#{name}".to_sym
+          after  = "after_#{name}".to_sym
+          if f = Common::AMQP.config.callbacks[before]
+            f.call(*args)
+          end
+
+          rs = yield if block_given?
+
+          if f = Common::AMQP.config.callbacks[after]
+            f.call(*args)
+          end
+
+          rs
+        end
+
+        def run_on_error_callback(e)
+          if f = Common::AMQP.config.callbacks[:on_error]
+            begin
+              f.call e
+            rescue Exception => e
+              $stderr.puts "ERROR on error callback: #{e.inspect}"
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/mixins/logger.rb

@@ -0,0 +1,17 @@
+module Vx
+  module Common
+    module AMQP
+      module Logger
+
+        %w{ debug info warn }.each do |m|
+          define_method m do |msg|
+            if log = Common::AMQP.logger
+              log.send(m, "[AMQP] #{msg}")
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/session.rb

@@ -0,0 +1,154 @@
+require 'bunny'
+require 'thread'
+
+module Vx
+  module Common
+    module AMQP
+      class Session
+
+        include Common::AMQP::Logger
+
+        CHANNEL_KEY = :vx_amqp_channel
+
+        @@session_lock = Mutex.new
+
+        attr_reader :conn
+
+        class << self
+          def shutdown
+            @shutdown = true
+          end
+
+          def shutdown?
+            @shutdown == true
+          end
+
+          def resume
+            @shutdown = false
+          end
+        end
+
+        def close
+          if open?
+            @@session_lock.synchronize do
+              info "closing connection"
+              begin
+                conn.close
+              rescue Bunny::ChannelError => e
+                warn e
+              end
+              info "wait..."
+              while conn.status != :closed
+                sleep 0.01
+              end
+              @conn = nil
+              info "connection closed"
+            end
+          end
+        end
+
+        def open
+          return self if open?
+
+          @@session_lock.synchronize do
+            self.class.resume
+
+            @conn ||= Bunny.new config.url, heartbeat: :server
+
+            unless conn.open?
+              info "connecting to #{conn_info}"
+              conn.start
+              info "wait connection to #{conn_info}"
+              while conn.connecting?
+                sleep 0.01
+              end
+              info "connected successfuly (#{server_name})"
+            end
+          end
+
+          self
+        end
+
+        def open?
+          conn && conn.open? && conn.status == :open
+        end
+
+        def declare_exchange(name, options = nil)
+          assert_connection_is_open
+
+          options  ||= {}
+          name     ||= config.default_exchange_name
+          ch         = options.delete(:channel) || channel
+          type, opts = get_exchange_type_and_options options
+          ch.exchange name, opts.merge(type: type)
+        end
+
+        def declare_queue(name, options = nil)
+          assert_connection_is_open
+
+          options ||= {}
+          ch = options.delete(:channel) || channel
+          name, opts = get_queue_name_and_options(name, options)
+          ch.queue name, opts
+        end
+
+        def channel
+          assert_connection_is_open
+
+          Thread.current[CHANNEL_KEY] || conn.default_channel
+        end
+
+        def with_channel
+          assert_connection_is_open
+
+          old,new = nil
+          begin
+            old,new = Thread.current[CHANNEL_KEY], conn.create_channel
+            Thread.current[CHANNEL_KEY] = new
+            yield
+          ensure
+            Thread.current[CHANNEL_KEY] = old
+            new.close if new && new.open?
+          end
+        end
+
+        def conn_info
+          if conn
+            "#{conn.user}:#{conn.host}:#{conn.port}/#{conn.vhost}"
+          end
+        end
+
+        def server_name
+          if conn
+            p = conn.server_properties || {}
+            "#{p["product"]}/#{p["version"]}"
+          end
+        end
+
+        def config
+          Common::AMQP.config
+        end
+
+        private
+
+          def get_exchange_type_and_options(options)
+            options = config.default_exchange_options.merge(options || {})
+            type = options.delete(:type) || config.default_exchange_type
+            [type, options]
+          end
+
+          def get_queue_name_and_options(name, options)
+            name  ||= AMQ::Protocol::EMPTY_STRING
+            [name, config.default_queue_options.merge(options || {})]
+          end
+
+          def assert_connection_is_open
+            open? || raise(ConnectionDoesNotExist.new "you need to run #{to_s}#open")
+          end
+
+          class ConnectionDoesNotExist < ::Exception ; end
+
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/supervisor/threaded.rb

@@ -0,0 +1,171 @@
+require 'thread'
+
+module Vx
+  module Common
+    module AMQP
+      class Supervisor::Threaded
+
+        include Common::AMQP::Logger
+        include Common::AMQP::Callbacks
+
+        POOL_INTERVAL = 0.5
+
+        Task = Struct.new(:object, :method, :id) do
+
+          attr_accessor :thread, :attempt, :start_at
+
+          def alive?
+            !!(thread && thread.alive?)
+          end
+
+          def inspect
+            %{#<Task
+                 object=#{object.to_s}
+                 method=#{method.inspect}
+                 id=#{id.inspect}
+                 alive=#{alive?}
+                 attempt=#{attempt}
+                 start_at=#{start_at}> }.gsub("\n", ' ').gsub(/ +/, ' ').strip
+          end
+        end
+
+        class SpawnAttemptsLimitReached < ::Exception ; end
+
+        class << self
+
+          @@shutdown = false
+
+          def build(tasks)
+            supervisor = new
+            tasks.each_pair do |k,v|
+              v.to_i.times do |n|
+                supervisor.add k, :subscribe, n
+              end
+            end
+            supervisor
+          end
+
+          def resume
+            @@shutdown = false
+          end
+
+          def shutdown?
+            @@shutdown
+          end
+
+          def shutdown
+            @@shutdown = true
+          end
+
+        end
+
+        def initialize
+          self.class.resume
+          @tasks = Array.new
+        end
+
+        def add(object, method, id)
+          @tasks.push Task.new(object, method, id).freeze
+        end
+
+        def size
+          @tasks.size
+        end
+
+        def shutdown?
+          self.class.shutdown?
+        end
+
+        def shutdown
+          self.class.shutdown
+        end
+
+        def run_async
+          Thread.new { run }.tap{|t| t.abort_on_exception = true }
+        end
+
+        def run
+          start_all_threads
+
+          loop do
+            task = @tasks.shift
+            break unless task
+
+            case
+            when shutdown?
+              log_thread_error task
+            when task.alive?
+              @tasks.push task
+            else
+              process_fail task
+            end
+
+            sleep POOL_INTERVAL unless shutdown?
+          end
+        end
+
+        private
+
+          def process_fail(task)
+            log_thread_error task
+            if check_attempt task
+              @tasks.push create_thread(task, task.attempt + 1)
+            else
+              raise SpawnAttemptsLimitReached
+            end
+          end
+
+          def start_all_threads
+            started_tasks = Array.new
+            while task = @tasks.shift
+              started_tasks.push create_thread(task, 0)
+            end
+            while task = started_tasks.shift
+              @tasks.push task
+            end
+          end
+
+          def create_thread(task, attempt)
+            attempt = 0 if reset_attempt?(task)
+            task.dup.tap do |new_task|
+              new_task.thread = Thread.new(new_task) do |t|
+                Thread.current[:vx_amqp_consumer_id] = t.id
+                t.object.send t.method
+              end
+              new_task.thread.abort_on_exception = false
+              new_task.attempt = attempt
+              new_task.start_at = Time.now
+              new_task.freeze
+              debug "spawn #{new_task.inspect}"
+            end
+          end
+
+          def log_thread_error(task)
+            return unless task.thread
+
+            begin
+              task.thread.value
+              nil
+            rescue Exception => e
+              STDERR.puts "#{e.inspect} in #{task.inspect}"
+              STDERR.puts e.backtrace.join("\n")
+              run_on_error_callback(e)
+              e
+            end
+          end
+
+          def reset_attempt?(task)
+            return true unless task.start_at
+
+            interval = 60
+            (task.start_at + interval) < Time.now
+          end
+
+          def check_attempt(task)
+            task.attempt.to_i <= Common::AMQP.config.spawn_attempts.to_i
+          end
+
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/testing.rb

@@ -0,0 +1,54 @@
+require File.expand_path("../../amqp", __FILE__)
+
+module Vx
+  module Common
+    module AMQP
+      module Testing
+
+        extend self
+
+        @@messages = Hash.new { |h,k| h[k] = [] }
+        @@messages_and_options = Hash.new { |h,k| h[k] = [] }
+
+        def messages
+          @@messages
+        end
+
+        def messages_and_options
+          @@messages_and_options
+        end
+
+        def clear
+          messages.clear
+          messages_and_options.clear
+        end
+      end
+
+      module Consumer::Publish
+        alias_method :real_publish, :publish
+
+        def publish(message, options = nil)
+          options ||= {}
+          Testing.messages[exchange_name] << message
+          Testing.messages_and_options[exchange_name] << [message, options]
+          self
+        end
+
+      end
+
+      module Consumer
+        module ClassMethods
+
+          def messages
+            Testing.messages[exchange_name]
+          end
+
+          def messages_and_options
+            Testing.messages_and_options[exchange_name]
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/vx/common/amqp/version.rb

@@ -0,0 +1,7 @@
+module Vx
+  module Common
+    module AMQP
+      VERSION = "0.2.6"
+    end
+  end
+end