active_publisher 1.1.1 → 1.2.0.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/active_publisher.gemspec +3 -0
- data/lib/active_publisher/async/in_memory_adapter/async_queue.rb +1 -1
- data/lib/active_publisher/async/redis_adapter/consumer.rb +39 -0
- data/lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb +98 -0
- data/lib/active_publisher/async/redis_adapter.rb +73 -0
- data/lib/active_publisher/configuration.rb +1 -0
- data/lib/active_publisher/version.rb +1 -1
- metadata +50 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: af572797671347ce2affc2ec5c7dff8b677157b9
|
4
|
+
data.tar.gz: 78780a6e31a452f4b61b0980850ca2ebccc26bd2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 556f432ec930501acef50a143cf02e11398e8bb8a38007d9126a0e3a27966fe873cba272e1ecbb8b3c7d231514f2e0b074cb593a6c7466c8681ecc3eb663424e
|
7
|
+
data.tar.gz: d92c6f495bd7c085c6b5995197d2dc9985c8e5e5a373358f75015ee7f59b3b7b66590920d166d9185d11fe9585a0dbad30077f3179a91129ced0ffb9eaf47cec
|
data/active_publisher.gemspec
CHANGED
@@ -30,7 +30,10 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.add_dependency 'concurrent-ruby'
|
31
31
|
spec.add_dependency 'multi_op_queue', '>= 0.2.0'
|
32
32
|
|
33
|
+
spec.add_development_dependency "benchmark-ips"
|
33
34
|
spec.add_development_dependency "bundler"
|
35
|
+
spec.add_development_dependency "connection_pool"
|
36
|
+
spec.add_development_dependency "fakeredis"
|
34
37
|
spec.add_development_dependency "pry"
|
35
38
|
spec.add_development_dependency "rake", "~> 10.0"
|
36
39
|
spec.add_development_dependency "rspec", "~> 3.2"
|
@@ -71,7 +71,7 @@ module ActivePublisher
|
|
71
71
|
|
72
72
|
# Check to see if we should restart the consumer.
|
73
73
|
if !consumer.alive? || consumer_is_lagging
|
74
|
-
consumer.kill
|
74
|
+
consumer.kill rescue nil
|
75
75
|
@consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
76
76
|
end
|
77
77
|
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require "active_publisher/async/redis_adapter/redis_multi_pop_queue"
|
2
|
+
|
3
|
+
module ActivePublisher
|
4
|
+
module Async
|
5
|
+
module RedisAdapter
|
6
|
+
class Consumer
|
7
|
+
SUPERVISOR_INTERVAL = {
|
8
|
+
:execution_interval => 10, # seconds
|
9
|
+
:timeout_interval => 5, # seconds
|
10
|
+
}
|
11
|
+
|
12
|
+
attr_reader :consumer, :queue, :supervisor
|
13
|
+
|
14
|
+
def initialize(redis_pool)
|
15
|
+
@queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
|
16
|
+
create_and_supervise_consumer!
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_and_supervise_consumer!
|
20
|
+
@consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
21
|
+
|
22
|
+
supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
|
23
|
+
# This may also be the place to start additional publishers when we are getting backed up ... ?
|
24
|
+
unless consumer.alive?
|
25
|
+
consumer.kill rescue nil
|
26
|
+
@consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Notify the current queue size.
|
30
|
+
::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
|
31
|
+
end
|
32
|
+
|
33
|
+
supervisor_task.execute
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module ActivePublisher
|
2
|
+
module Async
|
3
|
+
module RedisAdapter
|
4
|
+
class RedisMultiPopQueue
|
5
|
+
attr_reader :list_key, :redis_pool
|
6
|
+
|
7
|
+
def initialize(redis_connection_pool, new_list_key)
|
8
|
+
@redis_pool = redis_connection_pool
|
9
|
+
@list_key = new_list_key
|
10
|
+
end
|
11
|
+
|
12
|
+
def <<(message)
|
13
|
+
encoded_message = ::Marshal.dump(message)
|
14
|
+
|
15
|
+
redis_pool.with do |redis|
|
16
|
+
redis.lpush(list_key, encoded_message)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def concat(*messages)
|
21
|
+
messages = messages.flatten
|
22
|
+
messages.compact!
|
23
|
+
|
24
|
+
encoded_messages = []
|
25
|
+
messages.each do |message|
|
26
|
+
encoded_messages << ::Marshal.dump(message)
|
27
|
+
end
|
28
|
+
|
29
|
+
redis_pool.with do |redis|
|
30
|
+
redis.lpush(list_key, encoded_messages)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def empty?
|
35
|
+
size <= 0
|
36
|
+
end
|
37
|
+
|
38
|
+
def pop_up_to(num_to_pop, opts = {})
|
39
|
+
case opts
|
40
|
+
when TrueClass, FalseClass
|
41
|
+
non_bock = opts
|
42
|
+
when Hash
|
43
|
+
timeout = opts.fetch(:timeout, nil)
|
44
|
+
non_block = opts.fetch(:non_block, false)
|
45
|
+
end
|
46
|
+
|
47
|
+
queue_size = size
|
48
|
+
if queue_size <= 0
|
49
|
+
if non_block
|
50
|
+
raise ThreadError, "queue empty"
|
51
|
+
else
|
52
|
+
total_waited_time = 0
|
53
|
+
|
54
|
+
loop do
|
55
|
+
total_waited_time += 0.1
|
56
|
+
sleep 0.1
|
57
|
+
queue_size = size
|
58
|
+
|
59
|
+
if queue_size > 0
|
60
|
+
num_to_pop = [num_to_pop, queue_size].min # make sure we don't pop more than size
|
61
|
+
return shift(num_to_pop)
|
62
|
+
end
|
63
|
+
|
64
|
+
return if timeout && total_waited_time > timeout
|
65
|
+
end
|
66
|
+
end
|
67
|
+
else
|
68
|
+
shift(num_to_pop)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def shift(number)
|
73
|
+
messages = []
|
74
|
+
number = [number, size].min
|
75
|
+
redis_pool.with do |redis|
|
76
|
+
redis.pipelined do
|
77
|
+
number.times do
|
78
|
+
messages << redis.rpop(list_key)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
messages = [] if messages.nil?
|
84
|
+
messages = [messages] unless messages.respond_to?(:each)
|
85
|
+
messages.compact!
|
86
|
+
messages.map { |message| ::Marshal.load(message.value) }
|
87
|
+
end
|
88
|
+
|
89
|
+
def size
|
90
|
+
redis_pool.with do |redis|
|
91
|
+
redis.llen(list_key) || 0
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require "active_publisher/message"
|
2
|
+
require "active_publisher/async/redis_adapter/consumer"
|
3
|
+
require "multi_op_queue"
|
4
|
+
|
5
|
+
module ActivePublisher
|
6
|
+
module Async
|
7
|
+
module RedisAdapter
|
8
|
+
REDIS_LIST_KEY = "ACTIVE_PUBLISHER_LIST".freeze
|
9
|
+
|
10
|
+
def self.new(*args)
|
11
|
+
::ActivePublisher::Async::RedisAdapter::Adapter.new(*args)
|
12
|
+
end
|
13
|
+
|
14
|
+
class Adapter
|
15
|
+
SUPERVISOR_INTERVAL = {
|
16
|
+
:execution_interval => 1.5, # seconds
|
17
|
+
:timeout_interval => 1, # seconds
|
18
|
+
}
|
19
|
+
include ::ActivePublisher::Logging
|
20
|
+
|
21
|
+
attr_reader :async_queue, :redis_pool, :queue
|
22
|
+
|
23
|
+
def initialize(new_redis_pool)
|
24
|
+
logger.info "Starting redis publisher adapter"
|
25
|
+
# do something with supervision ?
|
26
|
+
@redis_pool = new_redis_pool
|
27
|
+
@async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
|
28
|
+
@queue = ::MultiOpQueue::Queue.new
|
29
|
+
|
30
|
+
supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
|
31
|
+
queue_size = queue.size
|
32
|
+
number_of_times = [queue_size / 50, 1].max # get the max number of times to flush
|
33
|
+
number_of_times = [number_of_times, 5].min # don't allow it to be more than 5 per run
|
34
|
+
number_of_times.times { flush_queue! }
|
35
|
+
end
|
36
|
+
|
37
|
+
supervisor_task.execute
|
38
|
+
end
|
39
|
+
|
40
|
+
def publish(route, payload, exchange_name, options = {})
|
41
|
+
message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
|
42
|
+
queue << ::Marshal.dump(message)
|
43
|
+
flush_queue! if queue.size >= 20 || options[:flush_queue]
|
44
|
+
|
45
|
+
nil
|
46
|
+
end
|
47
|
+
|
48
|
+
def shutdown!
|
49
|
+
logger.info "Draining async publisher redis adapter before shutdown."
|
50
|
+
flush_queue! until queue.empty?
|
51
|
+
# Sleeping 2.1 seconds because the most common redis `fsync` command in AOF mode is run every 1 second
|
52
|
+
# this will give at least 1 full `fsync` to run before the process dies
|
53
|
+
sleep 2.1
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def flush_queue!
|
59
|
+
return if queue.empty?
|
60
|
+
encoded_messages = queue.pop_up_to(25, :timeout => 0.001)
|
61
|
+
|
62
|
+
return if encoded_messages.nil?
|
63
|
+
return unless encoded_messages.respond_to?(:each)
|
64
|
+
return unless encoded_messages.size > 0
|
65
|
+
|
66
|
+
redis_pool.with do |redis|
|
67
|
+
redis.lpush(::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY, encoded_messages)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_publisher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0.pre
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Stien
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: exe
|
14
14
|
cert_chain: []
|
15
|
-
date: 2018-01-
|
15
|
+
date: 2018-01-16 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: bunny
|
@@ -70,6 +70,20 @@ dependencies:
|
|
70
70
|
- - ">="
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: 0.2.0
|
73
|
+
- !ruby/object:Gem::Dependency
|
74
|
+
name: benchmark-ips
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
type: :development
|
81
|
+
prerelease: false
|
82
|
+
version_requirements: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
73
87
|
- !ruby/object:Gem::Dependency
|
74
88
|
name: bundler
|
75
89
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,6 +98,34 @@ dependencies:
|
|
84
98
|
- - ">="
|
85
99
|
- !ruby/object:Gem::Version
|
86
100
|
version: '0'
|
101
|
+
- !ruby/object:Gem::Dependency
|
102
|
+
name: connection_pool
|
103
|
+
requirement: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - ">="
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: '0'
|
108
|
+
type: :development
|
109
|
+
prerelease: false
|
110
|
+
version_requirements: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
- !ruby/object:Gem::Dependency
|
116
|
+
name: fakeredis
|
117
|
+
requirement: !ruby/object:Gem::Requirement
|
118
|
+
requirements:
|
119
|
+
- - ">="
|
120
|
+
- !ruby/object:Gem::Version
|
121
|
+
version: '0'
|
122
|
+
type: :development
|
123
|
+
prerelease: false
|
124
|
+
version_requirements: !ruby/object:Gem::Requirement
|
125
|
+
requirements:
|
126
|
+
- - ">="
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '0'
|
87
129
|
- !ruby/object:Gem::Dependency
|
88
130
|
name: pry
|
89
131
|
requirement: !ruby/object:Gem::Requirement
|
@@ -154,6 +196,9 @@ files:
|
|
154
196
|
- lib/active_publisher/async/in_memory_adapter/async_queue.rb
|
155
197
|
- lib/active_publisher/async/in_memory_adapter/channel.rb
|
156
198
|
- lib/active_publisher/async/in_memory_adapter/consumer_thread.rb
|
199
|
+
- lib/active_publisher/async/redis_adapter.rb
|
200
|
+
- lib/active_publisher/async/redis_adapter/consumer.rb
|
201
|
+
- lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb
|
157
202
|
- lib/active_publisher/configuration.rb
|
158
203
|
- lib/active_publisher/connection.rb
|
159
204
|
- lib/active_publisher/logging.rb
|
@@ -174,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
174
219
|
version: '0'
|
175
220
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
176
221
|
requirements:
|
177
|
-
- - "
|
222
|
+
- - ">"
|
178
223
|
- !ruby/object:Gem::Version
|
179
|
-
version:
|
224
|
+
version: 1.3.1
|
180
225
|
requirements: []
|
181
226
|
rubyforge_project:
|
182
|
-
rubygems_version: 2.
|
227
|
+
rubygems_version: 2.5.1
|
183
228
|
signing_key:
|
184
229
|
specification_version: 4
|
185
230
|
summary: Aims to make publishing work across MRI and jRuby painless and add some nice
|