vx-lib-consumer 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0f2ade54a8f310c3b4f0548782bb2855117fdc3d
+  data.tar.gz: ca1b073eafd155833ae8ff603b425c369f0de5d6
+SHA512:
+  metadata.gz: 9f33d873c63cac1ea0c77cb3f0a8713cabd538fb52159965160dfa4d14069307d0ba0afc5b6ca22ac825d255f03c9c8bd060b280a5391ea6e4b3a3b717c0726e
+  data.tar.gz: eb06863cb980158841c173b42e52122f1606e9270f816f6968320473b78e678b602468b6c84653869ac9f7c5d694698b2a7502ecf9c3f39bccdcfefa208380b3

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+-fd
+--order=rand

data/.travis.yml

@@ -0,0 +1,7 @@
+services:
+  - rabbitmq
+
+rvm:
+  - 2.1
+
+script: bundle exec rspec spec/ -b

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in vx-consumer.gemspec
+gemspec
+
+group :test do
+  gem 'beefcake'
+  #gem 'memory_profiler'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Dmitry Galinsky
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# Vx::Lib::Consumer
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'vx-lib-consumer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install vx-consumer
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/vx-consumer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/vx/lib/consumer/ack.rb

@@ -0,0 +1,44 @@
+module Vx
+  module Lib
+    module Consumer
+      module Ack
+
+        def ack(multiple = false)
+          instrumentation = {
+            consumer:      self.class.params.consumer_name,
+            properties:    properties,
+            multiple:      multiple,
+            channel:       _channel.id
+          }
+          if _channel.open?
+            _channel.ack delivery_info.delivery_tag, multiple
+            instrument("ack", instrumentation)
+            true
+          else
+            instrument("ack_failed", instrumentation)
+            false
+          end
+        end
+
+        def nack(multiple = false, requeue = false)
+          instrumentation = {
+            consumer:      self.class.params.consumer_name,
+            properties:    properties,
+            multiple:      multiple,
+            requeue:       requeue,
+            channel:       channel.id
+          }
+          if _channel.open?
+            _channel.ack delivery_info.delivery_tag, multiple, requeue
+            instrument("nack", instrumentation)
+            true
+          else
+            instrument("nack_failed", instrumentation)
+            false
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/lib/consumer/configuration.rb

@@ -0,0 +1,69 @@
+require 'vx/common/rack/builder'
+
+module Vx
+  module Lib
+    module Consumer
+      class Configuration
+
+        DEBUG = 'VX_CONSUMER_DEBUG'.freeze
+
+
+        attr_accessor :default_exchange_options, :default_queue_options,
+          :default_publish_options, :default_exchange_type, :pool_timeout,
+          :heartbeat, :spawn_attempts, :content_type, :instrumenter, :debug,
+          :on_error, :builders, :prefetch
+
+        def initialize
+          reset!
+        end
+
+        def debug?
+          @debug ||= ENV[DEBUG]
+        end
+
+        def use(target, middleware, *args)
+          @builders[target].use middleware, *args
+        end
+
+        def on_error(&block)
+          @on_error = block if block
+          @on_error
+        end
+
+        def reset!
+          @default_exchange_type = :topic
+          @pool_timeout          = 0.5
+          @heartbeat             = :server
+
+          @spawn_attempts        = 1
+
+          @content_type          = 'application/json'
+          @prefetch              = 1
+
+          @instrumenter          = nil
+          @on_error              = ->(e, env){ nil }
+
+          @builders = {
+            pub: Vx::Common::Rack::Builder.new,
+            sub: Vx::Common::Rack::Builder.new
+          }
+
+          @default_exchange_options = {
+            durable:     true,
+            auto_delete: false
+          }
+
+          @default_queue_options = {
+            durable:      true,
+            auto_delete:  false,
+            exclusive:    false
+          }
+
+          @default_publish_options = {
+          }
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/lib/consumer/error.rb

@@ -0,0 +1,10 @@
+module Vx
+  module Lib
+    module Consumer
+
+      class ConnectionDoesNotExistError < StandardError ; end
+      class ModelIsNotdefined < StandardError ; end
+
+    end
+  end
+end

data/lib/vx/lib/consumer/instrument.rb

@@ -0,0 +1,27 @@
+module Vx
+  module Lib
+    module Consumer
+      module Instrument
+
+        def instrument(name, payload, &block)
+          name = "#{name}.consumer.vx".freeze
+
+          if Consumer.configuration.debug?
+            $stdout.puts " --> #{name}: #{payload}"
+          end
+
+          if Consumer.configuration.instrumenter
+            Consumer.configuration.instrumenter.instrument(name, payload, &block)
+          else
+            begin
+              yield if block_given?
+            rescue Exception => e
+              Consumer.handle_exception(e, {})
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/lib/consumer/params.rb

@@ -0,0 +1,80 @@
+module Vx
+  module Lib
+    module Consumer
+      Params = Struct.new(:consumer_class) do
+
+        attr_accessor :exchange_name, :exchange_options
+        attr_accessor :queue_name, :queue_options
+        attr_accessor :routing_key, :headers
+        attr_accessor :content_type
+        attr_accessor :ack
+        attr_accessor :exchange_type
+        attr_accessor :model
+
+        def exchange_name
+          @exchange_name || default_exchange_name
+        end
+
+        def queue_name
+          @queue_name || ""
+        end
+
+        def ack
+          !!@ack
+        end
+
+        def content_type
+          @content_type || config.content_type
+        end
+
+        def exchange_type
+          @exchange_type || config.default_exchange_type
+        end
+
+        def exchange_options
+          (@exchange_options || config.default_exchange_options).merge(type: exchange_type)
+        end
+
+        def queue_options
+          @queue_options || config.default_queue_options
+        end
+
+        def publish_options
+          config.default_publish_options
+        end
+
+        def bind_options
+          opts = { }
+          opts.merge!(routing_key: routing_key) if routing_key
+          opts.merge!(headers: headers) if headers
+          opts
+        end
+
+        def consumer_name
+          @consumer_name ||= consumer_class.to_s
+        end
+
+        def consumer_id
+          @consumer_id ||=
+            consumer_name.gsub(/::/, '/').
+              gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+              gsub(/([a-z\d])([A-Z])/,'\1_\2').
+              tr("-", "_").
+              downcase.
+              gsub("_consumer$", '')
+        end
+
+        private
+
+          def config
+            Consumer.configuration
+          end
+
+          def default_exchange_name
+            "amq.#{exchange_type}"
+          end
+
+      end
+    end
+  end
+end

data/lib/vx/lib/consumer/publish.rb

@@ -0,0 +1,47 @@
+require 'securerandom'
+
+module Vx
+  module Lib
+    module Consumer
+      module Publish
+
+        def publish(payload, options = {})
+          session.open
+
+          options ||= {}
+          options[:routing_key]  = params.routing_key if params.routing_key && !options.key?(:routing_key)
+          options[:headers]      = params.headers     if params.headers && !options.key?(:headers)
+
+          options[:content_type] ||= params.content_type || configuration.content_type
+          options[:message_id]   ||= SecureRandom.uuid
+
+          name = params.exchange_name
+
+          instrumentation = {
+            payload:     payload,
+            exchange:    name,
+            consumer:    params.consumer_name,
+            properties:  options,
+          }
+
+          with_middlewares :pub, instrumentation do
+            session.with_pub_channel do |ch|
+              instrument(:process_publishing, instrumentation.merge(channel: ch.id)) do
+                encoded = encode_payload(payload, options[:content_type])
+                x = session.declare_exchange ch, name, params.exchange_options
+                x.publish encoded, options
+              end
+            end
+          end
+        end
+
+        private
+
+          def encode_payload(payload, content_type)
+            Serializer.pack(content_type, payload)
+          end
+
+      end
+    end
+  end
+end

data/lib/vx/lib/consumer/rpc.rb

@@ -0,0 +1,217 @@
+require 'json'
+require 'thread'
+require 'securerandom'
+
+module Vx
+  module Lib
+    module Consumer
+      module Rpc
+
+        RPC_EXCHANGE_NAME  = "".freeze
+        JSON_CONTENT_TYPE  = 'application/json'.freeze
+        RPC_PAYLOAD_METHOD = 'method'.freeze
+        RPC_PAYLOAD_PARAMS = 'params'.freeze
+        RPC_PAYLOAD_RESULT = 'result'.freeze
+
+        class RpcProxy
+
+          attr_reader :client
+
+          def initialize(consumer)
+            @parent   = consumer
+            @client   = RpcClient.new consumer
+            @methods  = {}
+            @defined  = false
+          end
+
+          def call(method, args, options = {})
+            ns = @parent.params.consumer_id
+            @client.call ns, method, args, options
+          end
+
+          def define
+            @parent.exchange RPC_EXCHANGE_NAME
+            @parent.queue    "vx.rpc.#{@parent.params.consumer_id}".freeze, durable: false, auto_delete: true
+
+            @parent.send :define_method, :perform do |payload|
+              self.class.rpc.process_payload(properties, payload)
+            end
+
+            @defined = true
+          end
+
+          def action(name, fn)
+            define unless @defined
+            @methods[name.to_s] = fn
+          end
+
+          def process_payload(properties, payload)
+            m = payload[RPC_PAYLOAD_METHOD]
+            p = payload[RPC_PAYLOAD_PARAMS]
+
+            if fn = @methods[m]
+              re = fn.call(*p)
+              @parent.publish(
+                { 'result' => re },
+                routing_key:    properties[:reply_to],
+                correlation_id: properties[:correlation_id],
+                content_type:   JSON_CONTENT_TYPE
+              )
+            end
+          end
+        end
+
+        class RpcClient
+
+          REP    = "rep".freeze
+          REQ    = "req".freeze
+
+          attr_reader :consumer
+
+          def initialize(consumer)
+            @consumer = consumer
+            @consumed = false
+            @await    = {}
+            @mutex    = Mutex.new
+            @wakeup   = Mutex.new
+          end
+
+          def consume
+            return if @consumed
+
+            ch = consumer.session.conn.create_channel
+            consumer.session.assign_error_handlers_to_channel(ch)
+
+            @q = ch.queue(RPC_EXCHANGE_NAME, exclusive: true)
+
+            @subscriber =
+              @q.subscribe do |delivery_info, properties, payload|
+                handle_delivery ch, properties, payload
+              end
+            @consumed = true
+          end
+
+          def handle_delivery(ch, properties, payload)
+
+            if payload
+              payload = ::JSON.parse(payload)
+            end
+
+            instrumentation = {
+              consumer:   consumer.params.consumer_name,
+              queue:      @q.name,
+              rpc:        REP,
+              channel:    ch.id,
+              payload:    payload,
+              properties: properties
+            }
+
+            consumer.with_middlewares :sub, instrumentation do
+              consumer.instrument(:start_processing, instrumentation)
+              consumer.instrument(:process, instrumentation) do
+                call_id = properties[:correlation_id]
+                c = @mutex.synchronize{ @await.delete(call_id) }
+                if c
+                  @mutex.synchronize do
+                    @await[call_id] = [properties, payload]
+                  end
+                  @wakeup.synchronize do
+                    c.signal
+                  end
+                end
+              end
+            end
+          end
+
+          def call(ns, method, params, options = {})
+            timeout     = options[:timeout] || 3
+            routing_key = options[:routing_key] || "vx.rpc.#{ns}".freeze
+            call_id     = SecureRandom.uuid
+            cond        = ConditionVariable.new
+            result      = nil
+
+            message = {
+              method: method.to_s,
+              params: params,
+              id:     call_id
+            }
+
+            with_queue do |q|
+
+              consumer.session.with_pub_channel do |ch|
+                exch = ch.exchange RPC_EXCHANGE_NAME
+
+                instrumentation = {
+                  payload:     message,
+                  rpc:         REQ,
+                  exchange:    exch.name,
+                  consumer:    consumer.params.consumer_name,
+                  properties:  { routing_key: routing_key, correlation_id: call_id },
+                  channel:     ch.id
+                }
+
+                @mutex.synchronize { @await[call_id] = cond }
+
+                consumer.with_middlewares :pub, instrumentation do
+                  consumer.instrument(:process_publishing, instrumentation) do
+                    exch.publish(
+                      message.to_json,
+                      routing_key:    routing_key,
+                      correlation_id: call_id,
+                      reply_to:       q.name,
+                      content_type:   JSON_CONTENT_TYPE
+                    )
+                  end
+                end
+
+                @wakeup.synchronize{
+                  cond.wait(@wakeup, timeout)
+                }
+                @mutex.synchronize do
+                  _, payload = @await.delete(call_id)
+                  if payload
+                    result = payload[RPC_PAYLOAD_RESULT]
+                  else
+                    nil
+                  end
+                end
+              end
+            end
+
+            result
+
+          end
+
+          def with_queue
+            consume
+            yield @q
+          end
+
+          def subscriber?
+            !!@subscriber
+          end
+
+          def cancel
+            instrumentation = {
+              consumer: consumer.params.consumer_name,
+              rpc:      'consume'
+            }
+
+            consumer.instrument('cancel_consumer', instrumentation)
+
+            if subscriber?
+              @subscriber.cancel
+              @subscriber = nil
+              @consumed   = false
+            end
+          end
+        end
+
+        def rpc
+          @rpc ||= RpcProxy.new(self)
+        end
+
+      end
+    end
+  end
+end

data/lib/vx/lib/consumer/serializer.rb

@@ -0,0 +1,90 @@
+module Vx
+  module Lib
+    module Consumer
+      class Serializer
+        @@types = {}
+
+        Type = Struct.new(:content_type) do
+          def pack(&block)
+            @pack = block if block_given?
+            @pack
+          end
+
+          def unpack(&block)
+            @unpack = block if block_given?
+            @unpack
+          end
+        end
+
+        class << self
+          def types
+            @@types
+          end
+
+          def define(content_type, &block)
+            fmt = Type.new content_type
+            fmt.instance_eval(&block)
+            types.merge! content_type => fmt
+          end
+
+          def lookup(content_type)
+            types[content_type]
+          end
+
+          def pack(content_type, body)
+            if fmt = lookup(content_type)
+              fmt.pack.call(body)
+            end
+          end
+
+          def unpack(content_type, body, model)
+            if fmt = lookup(content_type)
+              fmt.unpack.call(body, model)
+            end
+          end
+        end
+
+        define 'text/plain' do
+          pack do |body|
+            body.to_s
+          end
+
+          unpack do |body, _|
+            body
+          end
+        end
+
+        define 'application/json' do
+          pack do |body|
+            if body.is_a?(String)
+              body
+            else
+              ::JSON.dump body
+            end
+          end
+
+          unpack do |payload, model|
+            if model && model.respond_to?(:from_json)
+              model.from_json payload
+            else
+              ::JSON.parse(payload)
+            end
+          end
+        end
+
+        define 'application/x-protobuf' do
+
+          pack do |object|
+            object.encode.to_s
+          end
+
+          unpack do |payload, model|
+            raise ModelIsNotDefined unless model
+            model.decode payload
+          end
+        end
+
+      end
+    end
+  end
+end