active_publisher 1.1.1 → 1.2.0.pre

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7f3edc99c354f7cf507f117d4ddfbb13974f3abc
4
- data.tar.gz: 32663bda945c4519d5767a205a32ea931eee689f
3
+ metadata.gz: af572797671347ce2affc2ec5c7dff8b677157b9
4
+ data.tar.gz: 78780a6e31a452f4b61b0980850ca2ebccc26bd2
5
5
  SHA512:
6
- metadata.gz: 0b08a1749006c13b88febda66c6b10b7fd5c6c6b1b8e4f84ee92a96fc2f7362c6d79c212b1481cc751126e30a228a2e6ef904f5b1128278f25e26a9035fc0331
7
- data.tar.gz: 7cd33f6f2960ca931fd8e40ff8a38d373b40479e70a09efe8cbb96bc6438771ca716c17bf88202ee32efbfaa4314a5b8ae36856102487a9976fb0ed340a3d1b6
6
+ metadata.gz: 556f432ec930501acef50a143cf02e11398e8bb8a38007d9126a0e3a27966fe873cba272e1ecbb8b3c7d231514f2e0b074cb593a6c7466c8681ecc3eb663424e
7
+ data.tar.gz: d92c6f495bd7c085c6b5995197d2dc9985c8e5e5a373358f75015ee7f59b3b7b66590920d166d9185d11fe9585a0dbad30077f3179a91129ced0ffb9eaf47cec
@@ -30,7 +30,10 @@ Gem::Specification.new do |spec|
30
30
  spec.add_dependency 'concurrent-ruby'
31
31
  spec.add_dependency 'multi_op_queue', '>= 0.2.0'
32
32
 
33
+ spec.add_development_dependency "benchmark-ips"
33
34
  spec.add_development_dependency "bundler"
35
+ spec.add_development_dependency "connection_pool"
36
+ spec.add_development_dependency "fakeredis"
34
37
  spec.add_development_dependency "pry"
35
38
  spec.add_development_dependency "rake", "~> 10.0"
36
39
  spec.add_development_dependency "rspec", "~> 3.2"
@@ -71,7 +71,7 @@ module ActivePublisher
71
71
 
72
72
  # Check to see if we should restart the consumer.
73
73
  if !consumer.alive? || consumer_is_lagging
74
- consumer.kill
74
+ consumer.kill rescue nil
75
75
  @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
76
76
  end
77
77
 
@@ -0,0 +1,39 @@
1
+ require "active_publisher/async/redis_adapter/redis_multi_pop_queue"
2
+
3
+ module ActivePublisher
4
+ module Async
5
+ module RedisAdapter
6
+ class Consumer
7
+ SUPERVISOR_INTERVAL = {
8
+ :execution_interval => 10, # seconds
9
+ :timeout_interval => 5, # seconds
10
+ }
11
+
12
+ attr_reader :consumer, :queue, :supervisor
13
+
14
+ def initialize(redis_pool)
15
+ @queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
16
+ create_and_supervise_consumer!
17
+ end
18
+
19
+ def create_and_supervise_consumer!
20
+ @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
21
+
22
+ supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
23
+ # This may also be the place to start additional publishers when we are getting backed up ... ?
24
+ unless consumer.alive?
25
+ consumer.kill rescue nil
26
+ @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
27
+ end
28
+
29
+ # Notify the current queue size.
30
+ ::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
31
+ end
32
+
33
+ supervisor_task.execute
34
+ end
35
+ end
36
+
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,98 @@
1
+ module ActivePublisher
2
+ module Async
3
+ module RedisAdapter
4
+ class RedisMultiPopQueue
5
+ attr_reader :list_key, :redis_pool
6
+
7
+ def initialize(redis_connection_pool, new_list_key)
8
+ @redis_pool = redis_connection_pool
9
+ @list_key = new_list_key
10
+ end
11
+
12
+ def <<(message)
13
+ encoded_message = ::Marshal.dump(message)
14
+
15
+ redis_pool.with do |redis|
16
+ redis.lpush(list_key, encoded_message)
17
+ end
18
+ end
19
+
20
+ def concat(*messages)
21
+ messages = messages.flatten
22
+ messages.compact!
23
+
24
+ encoded_messages = []
25
+ messages.each do |message|
26
+ encoded_messages << ::Marshal.dump(message)
27
+ end
28
+
29
+ redis_pool.with do |redis|
30
+ redis.lpush(list_key, encoded_messages)
31
+ end
32
+ end
33
+
34
+ def empty?
35
+ size <= 0
36
+ end
37
+
38
+ def pop_up_to(num_to_pop, opts = {})
39
+ case opts
40
+ when TrueClass, FalseClass
41
+ non_bock = opts
42
+ when Hash
43
+ timeout = opts.fetch(:timeout, nil)
44
+ non_block = opts.fetch(:non_block, false)
45
+ end
46
+
47
+ queue_size = size
48
+ if queue_size <= 0
49
+ if non_block
50
+ raise ThreadError, "queue empty"
51
+ else
52
+ total_waited_time = 0
53
+
54
+ loop do
55
+ total_waited_time += 0.1
56
+ sleep 0.1
57
+ queue_size = size
58
+
59
+ if queue_size > 0
60
+ num_to_pop = [num_to_pop, queue_size].min # make sure we don't pop more than size
61
+ return shift(num_to_pop)
62
+ end
63
+
64
+ return if timeout && total_waited_time > timeout
65
+ end
66
+ end
67
+ else
68
+ shift(num_to_pop)
69
+ end
70
+ end
71
+
72
+ def shift(number)
73
+ messages = []
74
+ number = [number, size].min
75
+ redis_pool.with do |redis|
76
+ redis.pipelined do
77
+ number.times do
78
+ messages << redis.rpop(list_key)
79
+ end
80
+ end
81
+ end
82
+
83
+ messages = [] if messages.nil?
84
+ messages = [messages] unless messages.respond_to?(:each)
85
+ messages.compact!
86
+ messages.map { |message| ::Marshal.load(message.value) }
87
+ end
88
+
89
+ def size
90
+ redis_pool.with do |redis|
91
+ redis.llen(list_key) || 0
92
+ end
93
+ end
94
+ end
95
+
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,73 @@
1
+ require "active_publisher/message"
2
+ require "active_publisher/async/redis_adapter/consumer"
3
+ require "multi_op_queue"
4
+
5
+ module ActivePublisher
6
+ module Async
7
+ module RedisAdapter
8
+ REDIS_LIST_KEY = "ACTIVE_PUBLISHER_LIST".freeze
9
+
10
+ def self.new(*args)
11
+ ::ActivePublisher::Async::RedisAdapter::Adapter.new(*args)
12
+ end
13
+
14
+ class Adapter
15
+ SUPERVISOR_INTERVAL = {
16
+ :execution_interval => 1.5, # seconds
17
+ :timeout_interval => 1, # seconds
18
+ }
19
+ include ::ActivePublisher::Logging
20
+
21
+ attr_reader :async_queue, :redis_pool, :queue
22
+
23
+ def initialize(new_redis_pool)
24
+ logger.info "Starting redis publisher adapter"
25
+ # do something with supervision ?
26
+ @redis_pool = new_redis_pool
27
+ @async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
28
+ @queue = ::MultiOpQueue::Queue.new
29
+
30
+ supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
31
+ queue_size = queue.size
32
+ number_of_times = [queue_size / 50, 1].max # get the max number of times to flush
33
+ number_of_times = [number_of_times, 5].min # don't allow it to be more than 5 per run
34
+ number_of_times.times { flush_queue! }
35
+ end
36
+
37
+ supervisor_task.execute
38
+ end
39
+
40
+ def publish(route, payload, exchange_name, options = {})
41
+ message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
42
+ queue << ::Marshal.dump(message)
43
+ flush_queue! if queue.size >= 20 || options[:flush_queue]
44
+
45
+ nil
46
+ end
47
+
48
+ def shutdown!
49
+ logger.info "Draining async publisher redis adapter before shutdown."
50
+ flush_queue! until queue.empty?
51
+ # Sleeping 2.1 seconds because the most common redis `fsync` command in AOF mode is run every 1 second
52
+ # this will give at least 1 full `fsync` to run before the process dies
53
+ sleep 2.1
54
+ end
55
+
56
+ private
57
+
58
+ def flush_queue!
59
+ return if queue.empty?
60
+ encoded_messages = queue.pop_up_to(25, :timeout => 0.001)
61
+
62
+ return if encoded_messages.nil?
63
+ return unless encoded_messages.respond_to?(:each)
64
+ return unless encoded_messages.size > 0
65
+
66
+ redis_pool.with do |redis|
67
+ redis.lpush(::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY, encoded_messages)
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -1,4 +1,5 @@
1
1
  require "yaml"
2
+ require "erb"
2
3
 
3
4
  module ActivePublisher
4
5
  class Configuration
@@ -1,3 +1,3 @@
1
1
  module ActivePublisher
2
- VERSION = "1.1.1"
2
+ VERSION = "1.2.0.pre"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: active_publisher
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.1
4
+ version: 1.2.0.pre
5
5
  platform: ruby
6
6
  authors:
7
7
  - Brian Stien
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: exe
14
14
  cert_chain: []
15
- date: 2018-01-11 00:00:00.000000000 Z
15
+ date: 2018-01-16 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: bunny
@@ -70,6 +70,20 @@ dependencies:
70
70
  - - ">="
71
71
  - !ruby/object:Gem::Version
72
72
  version: 0.2.0
73
+ - !ruby/object:Gem::Dependency
74
+ name: benchmark-ips
75
+ requirement: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - ">="
78
+ - !ruby/object:Gem::Version
79
+ version: '0'
80
+ type: :development
81
+ prerelease: false
82
+ version_requirements: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
73
87
  - !ruby/object:Gem::Dependency
74
88
  name: bundler
75
89
  requirement: !ruby/object:Gem::Requirement
@@ -84,6 +98,34 @@ dependencies:
84
98
  - - ">="
85
99
  - !ruby/object:Gem::Version
86
100
  version: '0'
101
+ - !ruby/object:Gem::Dependency
102
+ name: connection_pool
103
+ requirement: !ruby/object:Gem::Requirement
104
+ requirements:
105
+ - - ">="
106
+ - !ruby/object:Gem::Version
107
+ version: '0'
108
+ type: :development
109
+ prerelease: false
110
+ version_requirements: !ruby/object:Gem::Requirement
111
+ requirements:
112
+ - - ">="
113
+ - !ruby/object:Gem::Version
114
+ version: '0'
115
+ - !ruby/object:Gem::Dependency
116
+ name: fakeredis
117
+ requirement: !ruby/object:Gem::Requirement
118
+ requirements:
119
+ - - ">="
120
+ - !ruby/object:Gem::Version
121
+ version: '0'
122
+ type: :development
123
+ prerelease: false
124
+ version_requirements: !ruby/object:Gem::Requirement
125
+ requirements:
126
+ - - ">="
127
+ - !ruby/object:Gem::Version
128
+ version: '0'
87
129
  - !ruby/object:Gem::Dependency
88
130
  name: pry
89
131
  requirement: !ruby/object:Gem::Requirement
@@ -154,6 +196,9 @@ files:
154
196
  - lib/active_publisher/async/in_memory_adapter/async_queue.rb
155
197
  - lib/active_publisher/async/in_memory_adapter/channel.rb
156
198
  - lib/active_publisher/async/in_memory_adapter/consumer_thread.rb
199
+ - lib/active_publisher/async/redis_adapter.rb
200
+ - lib/active_publisher/async/redis_adapter/consumer.rb
201
+ - lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb
157
202
  - lib/active_publisher/configuration.rb
158
203
  - lib/active_publisher/connection.rb
159
204
  - lib/active_publisher/logging.rb
@@ -174,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
174
219
  version: '0'
175
220
  required_rubygems_version: !ruby/object:Gem::Requirement
176
221
  requirements:
177
- - - ">="
222
+ - - ">"
178
223
  - !ruby/object:Gem::Version
179
- version: '0'
224
+ version: 1.3.1
180
225
  requirements: []
181
226
  rubyforge_project:
182
- rubygems_version: 2.6.13
227
+ rubygems_version: 2.5.1
183
228
  signing_key:
184
229
  specification_version: 4
185
230
  summary: Aims to make publishing work across MRI and jRuby painless and add some nice