vx-lib-consumer 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 0f2ade54a8f310c3b4f0548782bb2855117fdc3d
4
+ data.tar.gz: ca1b073eafd155833ae8ff603b425c369f0de5d6
5
+ SHA512:
6
+ metadata.gz: 9f33d873c63cac1ea0c77cb3f0a8713cabd538fb52159965160dfa4d14069307d0ba0afc5b6ca22ac825d255f03c9c8bd060b280a5391ea6e4b3a3b717c0726e
7
+ data.tar.gz: eb06863cb980158841c173b42e52122f1606e9270f816f6968320473b78e678b602468b6c84653869ac9f7c5d694698b2a7502ecf9c3f39bccdcfefa208380b3
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --color
2
+ -fd
3
+ --order=rand
data/.travis.yml ADDED
@@ -0,0 +1,7 @@
1
+ services:
2
+ - rabbitmq
3
+
4
+ rvm:
5
+ - 2.1
6
+
7
+ script: bundle exec rspec spec/ -b
data/Gemfile ADDED
@@ -0,0 +1,9 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in vx-consumer.gemspec
4
+ gemspec
5
+
6
+ group :test do
7
+ gem 'beefcake'
8
+ #gem 'memory_profiler'
9
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Dmitry Galinsky
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,30 @@
1
+ # Vx::Lib::Consumer
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'vx-lib-consumer'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install vx-consumer
18
+
19
+ ## Usage
20
+
21
+ TODO: Write usage instructions here
22
+
23
+ ## Contributing
24
+
25
+ 1. Fork it ( http://github.com/<my-github-username>/vx-consumer/fork )
26
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
27
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
28
+ 4. Push to the branch (`git push origin my-new-feature`)
29
+ 5. Create new Pull Request
30
+
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,44 @@
1
+ module Vx
2
+ module Lib
3
+ module Consumer
4
+ module Ack
5
+
6
+ def ack(multiple = false)
7
+ instrumentation = {
8
+ consumer: self.class.params.consumer_name,
9
+ properties: properties,
10
+ multiple: multiple,
11
+ channel: _channel.id
12
+ }
13
+ if _channel.open?
14
+ _channel.ack delivery_info.delivery_tag, multiple
15
+ instrument("ack", instrumentation)
16
+ true
17
+ else
18
+ instrument("ack_failed", instrumentation)
19
+ false
20
+ end
21
+ end
22
+
23
+ def nack(multiple = false, requeue = false)
24
+ instrumentation = {
25
+ consumer: self.class.params.consumer_name,
26
+ properties: properties,
27
+ multiple: multiple,
28
+ requeue: requeue,
29
+ channel: channel.id
30
+ }
31
+ if _channel.open?
32
+ _channel.ack delivery_info.delivery_tag, multiple, requeue
33
+ instrument("nack", instrumentation)
34
+ true
35
+ else
36
+ instrument("nack_failed", instrumentation)
37
+ false
38
+ end
39
+ end
40
+
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,69 @@
1
+ require 'vx/common/rack/builder'
2
+
3
+ module Vx
4
+ module Lib
5
+ module Consumer
6
+ class Configuration
7
+
8
+ DEBUG = 'VX_CONSUMER_DEBUG'.freeze
9
+
10
+
11
+ attr_accessor :default_exchange_options, :default_queue_options,
12
+ :default_publish_options, :default_exchange_type, :pool_timeout,
13
+ :heartbeat, :spawn_attempts, :content_type, :instrumenter, :debug,
14
+ :on_error, :builders, :prefetch
15
+
16
+ def initialize
17
+ reset!
18
+ end
19
+
20
+ def debug?
21
+ @debug ||= ENV[DEBUG]
22
+ end
23
+
24
+ def use(target, middleware, *args)
25
+ @builders[target].use middleware, *args
26
+ end
27
+
28
+ def on_error(&block)
29
+ @on_error = block if block
30
+ @on_error
31
+ end
32
+
33
+ def reset!
34
+ @default_exchange_type = :topic
35
+ @pool_timeout = 0.5
36
+ @heartbeat = :server
37
+
38
+ @spawn_attempts = 1
39
+
40
+ @content_type = 'application/json'
41
+ @prefetch = 1
42
+
43
+ @instrumenter = nil
44
+ @on_error = ->(e, env){ nil }
45
+
46
+ @builders = {
47
+ pub: Vx::Common::Rack::Builder.new,
48
+ sub: Vx::Common::Rack::Builder.new
49
+ }
50
+
51
+ @default_exchange_options = {
52
+ durable: true,
53
+ auto_delete: false
54
+ }
55
+
56
+ @default_queue_options = {
57
+ durable: true,
58
+ auto_delete: false,
59
+ exclusive: false
60
+ }
61
+
62
+ @default_publish_options = {
63
+ }
64
+ end
65
+
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,10 @@
1
+ module Vx
2
+ module Lib
3
+ module Consumer
4
+
5
+ class ConnectionDoesNotExistError < StandardError ; end
6
+ class ModelIsNotdefined < StandardError ; end
7
+
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,27 @@
1
+ module Vx
2
+ module Lib
3
+ module Consumer
4
+ module Instrument
5
+
6
+ def instrument(name, payload, &block)
7
+ name = "#{name}.consumer.vx".freeze
8
+
9
+ if Consumer.configuration.debug?
10
+ $stdout.puts " --> #{name}: #{payload}"
11
+ end
12
+
13
+ if Consumer.configuration.instrumenter
14
+ Consumer.configuration.instrumenter.instrument(name, payload, &block)
15
+ else
16
+ begin
17
+ yield if block_given?
18
+ rescue Exception => e
19
+ Consumer.handle_exception(e, {})
20
+ end
21
+ end
22
+ end
23
+
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,80 @@
1
+ module Vx
2
+ module Lib
3
+ module Consumer
4
+ Params = Struct.new(:consumer_class) do
5
+
6
+ attr_accessor :exchange_name, :exchange_options
7
+ attr_accessor :queue_name, :queue_options
8
+ attr_accessor :routing_key, :headers
9
+ attr_accessor :content_type
10
+ attr_accessor :ack
11
+ attr_accessor :exchange_type
12
+ attr_accessor :model
13
+
14
+ def exchange_name
15
+ @exchange_name || default_exchange_name
16
+ end
17
+
18
+ def queue_name
19
+ @queue_name || ""
20
+ end
21
+
22
+ def ack
23
+ !!@ack
24
+ end
25
+
26
+ def content_type
27
+ @content_type || config.content_type
28
+ end
29
+
30
+ def exchange_type
31
+ @exchange_type || config.default_exchange_type
32
+ end
33
+
34
+ def exchange_options
35
+ (@exchange_options || config.default_exchange_options).merge(type: exchange_type)
36
+ end
37
+
38
+ def queue_options
39
+ @queue_options || config.default_queue_options
40
+ end
41
+
42
+ def publish_options
43
+ config.default_publish_options
44
+ end
45
+
46
+ def bind_options
47
+ opts = { }
48
+ opts.merge!(routing_key: routing_key) if routing_key
49
+ opts.merge!(headers: headers) if headers
50
+ opts
51
+ end
52
+
53
+ def consumer_name
54
+ @consumer_name ||= consumer_class.to_s
55
+ end
56
+
57
+ def consumer_id
58
+ @consumer_id ||=
59
+ consumer_name.gsub(/::/, '/').
60
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
61
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
62
+ tr("-", "_").
63
+ downcase.
64
+ gsub("_consumer$", '')
65
+ end
66
+
67
+ private
68
+
69
+ def config
70
+ Consumer.configuration
71
+ end
72
+
73
+ def default_exchange_name
74
+ "amq.#{exchange_type}"
75
+ end
76
+
77
+ end
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,47 @@
1
+ require 'securerandom'
2
+
3
+ module Vx
4
+ module Lib
5
+ module Consumer
6
+ module Publish
7
+
8
+ def publish(payload, options = {})
9
+ session.open
10
+
11
+ options ||= {}
12
+ options[:routing_key] = params.routing_key if params.routing_key && !options.key?(:routing_key)
13
+ options[:headers] = params.headers if params.headers && !options.key?(:headers)
14
+
15
+ options[:content_type] ||= params.content_type || configuration.content_type
16
+ options[:message_id] ||= SecureRandom.uuid
17
+
18
+ name = params.exchange_name
19
+
20
+ instrumentation = {
21
+ payload: payload,
22
+ exchange: name,
23
+ consumer: params.consumer_name,
24
+ properties: options,
25
+ }
26
+
27
+ with_middlewares :pub, instrumentation do
28
+ session.with_pub_channel do |ch|
29
+ instrument(:process_publishing, instrumentation.merge(channel: ch.id)) do
30
+ encoded = encode_payload(payload, options[:content_type])
31
+ x = session.declare_exchange ch, name, params.exchange_options
32
+ x.publish encoded, options
33
+ end
34
+ end
35
+ end
36
+ end
37
+
38
+ private
39
+
40
+ def encode_payload(payload, content_type)
41
+ Serializer.pack(content_type, payload)
42
+ end
43
+
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,217 @@
1
+ require 'json'
2
+ require 'thread'
3
+ require 'securerandom'
4
+
5
+ module Vx
6
+ module Lib
7
+ module Consumer
8
+ module Rpc
9
+
10
+ RPC_EXCHANGE_NAME = "".freeze
11
+ JSON_CONTENT_TYPE = 'application/json'.freeze
12
+ RPC_PAYLOAD_METHOD = 'method'.freeze
13
+ RPC_PAYLOAD_PARAMS = 'params'.freeze
14
+ RPC_PAYLOAD_RESULT = 'result'.freeze
15
+
16
+ class RpcProxy
17
+
18
+ attr_reader :client
19
+
20
+ def initialize(consumer)
21
+ @parent = consumer
22
+ @client = RpcClient.new consumer
23
+ @methods = {}
24
+ @defined = false
25
+ end
26
+
27
+ def call(method, args, options = {})
28
+ ns = @parent.params.consumer_id
29
+ @client.call ns, method, args, options
30
+ end
31
+
32
+ def define
33
+ @parent.exchange RPC_EXCHANGE_NAME
34
+ @parent.queue "vx.rpc.#{@parent.params.consumer_id}".freeze, durable: false, auto_delete: true
35
+
36
+ @parent.send :define_method, :perform do |payload|
37
+ self.class.rpc.process_payload(properties, payload)
38
+ end
39
+
40
+ @defined = true
41
+ end
42
+
43
+ def action(name, fn)
44
+ define unless @defined
45
+ @methods[name.to_s] = fn
46
+ end
47
+
48
+ def process_payload(properties, payload)
49
+ m = payload[RPC_PAYLOAD_METHOD]
50
+ p = payload[RPC_PAYLOAD_PARAMS]
51
+
52
+ if fn = @methods[m]
53
+ re = fn.call(*p)
54
+ @parent.publish(
55
+ { 'result' => re },
56
+ routing_key: properties[:reply_to],
57
+ correlation_id: properties[:correlation_id],
58
+ content_type: JSON_CONTENT_TYPE
59
+ )
60
+ end
61
+ end
62
+ end
63
+
64
+ class RpcClient
65
+
66
+ REP = "rep".freeze
67
+ REQ = "req".freeze
68
+
69
+ attr_reader :consumer
70
+
71
+ def initialize(consumer)
72
+ @consumer = consumer
73
+ @consumed = false
74
+ @await = {}
75
+ @mutex = Mutex.new
76
+ @wakeup = Mutex.new
77
+ end
78
+
79
+ def consume
80
+ return if @consumed
81
+
82
+ ch = consumer.session.conn.create_channel
83
+ consumer.session.assign_error_handlers_to_channel(ch)
84
+
85
+ @q = ch.queue(RPC_EXCHANGE_NAME, exclusive: true)
86
+
87
+ @subscriber =
88
+ @q.subscribe do |delivery_info, properties, payload|
89
+ handle_delivery ch, properties, payload
90
+ end
91
+ @consumed = true
92
+ end
93
+
94
+ def handle_delivery(ch, properties, payload)
95
+
96
+ if payload
97
+ payload = ::JSON.parse(payload)
98
+ end
99
+
100
+ instrumentation = {
101
+ consumer: consumer.params.consumer_name,
102
+ queue: @q.name,
103
+ rpc: REP,
104
+ channel: ch.id,
105
+ payload: payload,
106
+ properties: properties
107
+ }
108
+
109
+ consumer.with_middlewares :sub, instrumentation do
110
+ consumer.instrument(:start_processing, instrumentation)
111
+ consumer.instrument(:process, instrumentation) do
112
+ call_id = properties[:correlation_id]
113
+ c = @mutex.synchronize{ @await.delete(call_id) }
114
+ if c
115
+ @mutex.synchronize do
116
+ @await[call_id] = [properties, payload]
117
+ end
118
+ @wakeup.synchronize do
119
+ c.signal
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
125
+
126
+ def call(ns, method, params, options = {})
127
+ timeout = options[:timeout] || 3
128
+ routing_key = options[:routing_key] || "vx.rpc.#{ns}".freeze
129
+ call_id = SecureRandom.uuid
130
+ cond = ConditionVariable.new
131
+ result = nil
132
+
133
+ message = {
134
+ method: method.to_s,
135
+ params: params,
136
+ id: call_id
137
+ }
138
+
139
+ with_queue do |q|
140
+
141
+ consumer.session.with_pub_channel do |ch|
142
+ exch = ch.exchange RPC_EXCHANGE_NAME
143
+
144
+ instrumentation = {
145
+ payload: message,
146
+ rpc: REQ,
147
+ exchange: exch.name,
148
+ consumer: consumer.params.consumer_name,
149
+ properties: { routing_key: routing_key, correlation_id: call_id },
150
+ channel: ch.id
151
+ }
152
+
153
+ @mutex.synchronize { @await[call_id] = cond }
154
+
155
+ consumer.with_middlewares :pub, instrumentation do
156
+ consumer.instrument(:process_publishing, instrumentation) do
157
+ exch.publish(
158
+ message.to_json,
159
+ routing_key: routing_key,
160
+ correlation_id: call_id,
161
+ reply_to: q.name,
162
+ content_type: JSON_CONTENT_TYPE
163
+ )
164
+ end
165
+ end
166
+
167
+ @wakeup.synchronize{
168
+ cond.wait(@wakeup, timeout)
169
+ }
170
+ @mutex.synchronize do
171
+ _, payload = @await.delete(call_id)
172
+ if payload
173
+ result = payload[RPC_PAYLOAD_RESULT]
174
+ else
175
+ nil
176
+ end
177
+ end
178
+ end
179
+ end
180
+
181
+ result
182
+
183
+ end
184
+
185
+ def with_queue
186
+ consume
187
+ yield @q
188
+ end
189
+
190
+ def subscriber?
191
+ !!@subscriber
192
+ end
193
+
194
+ def cancel
195
+ instrumentation = {
196
+ consumer: consumer.params.consumer_name,
197
+ rpc: 'consume'
198
+ }
199
+
200
+ consumer.instrument('cancel_consumer', instrumentation)
201
+
202
+ if subscriber?
203
+ @subscriber.cancel
204
+ @subscriber = nil
205
+ @consumed = false
206
+ end
207
+ end
208
+ end
209
+
210
+ def rpc
211
+ @rpc ||= RpcProxy.new(self)
212
+ end
213
+
214
+ end
215
+ end
216
+ end
217
+ end
@@ -0,0 +1,90 @@
1
+ module Vx
2
+ module Lib
3
+ module Consumer
4
+ class Serializer
5
+ @@types = {}
6
+
7
+ Type = Struct.new(:content_type) do
8
+ def pack(&block)
9
+ @pack = block if block_given?
10
+ @pack
11
+ end
12
+
13
+ def unpack(&block)
14
+ @unpack = block if block_given?
15
+ @unpack
16
+ end
17
+ end
18
+
19
+ class << self
20
+ def types
21
+ @@types
22
+ end
23
+
24
+ def define(content_type, &block)
25
+ fmt = Type.new content_type
26
+ fmt.instance_eval(&block)
27
+ types.merge! content_type => fmt
28
+ end
29
+
30
+ def lookup(content_type)
31
+ types[content_type]
32
+ end
33
+
34
+ def pack(content_type, body)
35
+ if fmt = lookup(content_type)
36
+ fmt.pack.call(body)
37
+ end
38
+ end
39
+
40
+ def unpack(content_type, body, model)
41
+ if fmt = lookup(content_type)
42
+ fmt.unpack.call(body, model)
43
+ end
44
+ end
45
+ end
46
+
47
+ define 'text/plain' do
48
+ pack do |body|
49
+ body.to_s
50
+ end
51
+
52
+ unpack do |body, _|
53
+ body
54
+ end
55
+ end
56
+
57
+ define 'application/json' do
58
+ pack do |body|
59
+ if body.is_a?(String)
60
+ body
61
+ else
62
+ ::JSON.dump body
63
+ end
64
+ end
65
+
66
+ unpack do |payload, model|
67
+ if model && model.respond_to?(:from_json)
68
+ model.from_json payload
69
+ else
70
+ ::JSON.parse(payload)
71
+ end
72
+ end
73
+ end
74
+
75
+ define 'application/x-protobuf' do
76
+
77
+ pack do |object|
78
+ object.encode.to_s
79
+ end
80
+
81
+ unpack do |payload, model|
82
+ raise ModelIsNotDefined unless model
83
+ model.decode payload
84
+ end
85
+ end
86
+
87
+ end
88
+ end
89
+ end
90
+ end