active_publisher 1.2.6 → 1.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/lib/active_publisher/async/in_memory_adapter/async_queue.rb +30 -21
- data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb +3 -1
- data/lib/active_publisher/async/redis_adapter/consumer.rb +19 -14
- data/lib/active_publisher/configuration.rb +2 -0
- data/lib/active_publisher/version.rb +1 -1
- data/lib/active_publisher.rb +1 -0
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e205892cb22a44c4a5f8792b52ccd2d9d1f91cee827c86b26caf928b6790a2e
|
4
|
+
data.tar.gz: 3b868e1c64ce26f081911c6cf70ed3db5e3a9adc75beb844fea3db8adc2484f3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c0285683ccd9ca51a7c9b571cb0b47db2ef79404e344d7759cd3c69ea9eabb108cbbc5e5146b084fef5b28d90f6876f2c0fda8c7dba7f81cbfb99bb2e5b98b6d
|
7
|
+
data.tar.gz: bab6a2b9b9a2e785d9cfcf2523219ab2e4662350bcc41eefa807bc11312f1f0024ce01952fc62eb9bf60ca362bc71c4892551b6caa0b8bba4e6744776c42f991
|
data/README.md
CHANGED
@@ -55,7 +55,9 @@ Defaults for the configuration are:
|
|
55
55
|
:network_recovery_interval => 1,
|
56
56
|
:password => "guest",
|
57
57
|
:port => 5672,
|
58
|
+
:publisher_threads => 1,
|
58
59
|
:publisher_confirms => false,
|
60
|
+
:publisher_confirms_timeout => 5_000,
|
59
61
|
:seconds_to_wait_for_graceful_shutdown => 30,
|
60
62
|
:timeout => 1,
|
61
63
|
:tls => false,
|
@@ -14,14 +14,15 @@ module ActivePublisher
|
|
14
14
|
:max_queue_size,
|
15
15
|
:supervisor_interval
|
16
16
|
|
17
|
-
attr_reader :
|
17
|
+
attr_reader :consumers, :queue, :supervisor
|
18
18
|
|
19
19
|
def initialize(back_pressure_strategy, max_queue_size, supervisor_interval)
|
20
20
|
self.back_pressure_strategy = back_pressure_strategy
|
21
21
|
@max_queue_size = max_queue_size
|
22
22
|
@supervisor_interval = supervisor_interval
|
23
23
|
@queue = ::MultiOpQueue::Queue.new
|
24
|
-
|
24
|
+
@consumers = {}
|
25
|
+
create_and_supervise_consumers!
|
25
26
|
end
|
26
27
|
|
27
28
|
def back_pressure_strategy=(strategy)
|
@@ -52,36 +53,44 @@ module ActivePublisher
|
|
52
53
|
def size
|
53
54
|
# Requests might be in flight (out of the queue, but not yet published), so taking the max should be
|
54
55
|
# good enough to make sure we're honest about the actual queue size.
|
55
|
-
return queue.size if
|
56
|
-
[queue.size,
|
56
|
+
return queue.size if consumers.empty?
|
57
|
+
[queue.size, consumer_sampled_queue_size].max
|
57
58
|
end
|
58
59
|
|
59
60
|
private
|
60
61
|
|
61
|
-
def
|
62
|
-
|
62
|
+
def create_and_supervise_consumers!
|
63
|
+
::ActivePublisher.configuration.publisher_threads.times do
|
64
|
+
consumer_id = ::SecureRandom.uuid
|
65
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
66
|
+
supervisor_task = ::Concurrent::TimerTask.new(:execution_interval => supervisor_interval) do
|
67
|
+
current_time = ::Time.now
|
68
|
+
consumer = consumers[consumer_id]
|
63
69
|
|
64
|
-
|
65
|
-
|
70
|
+
# Consumer is lagging if it does not "tick" at least once every 10 seconds.
|
71
|
+
seconds_since_last_tick = current_time - consumer.last_tick_at
|
72
|
+
consumer_is_lagging = seconds_since_last_tick > ::ActivePublisher.configuration.max_async_publisher_lag_time
|
73
|
+
logger.error "ActivePublisher consumer is lagging. Last consumer tick was #{seconds_since_last_tick} seconds ago." if consumer_is_lagging
|
66
74
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
75
|
+
# Check to see if we should restart the consumer.
|
76
|
+
if !consumer.alive? || consumer_is_lagging
|
77
|
+
consumer.kill rescue nil
|
78
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
79
|
+
::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
|
80
|
+
end
|
71
81
|
|
72
|
-
|
73
|
-
|
74
|
-
consumer.kill rescue nil
|
75
|
-
@consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
82
|
+
# Notify the current queue size.
|
83
|
+
::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
|
76
84
|
end
|
77
|
-
|
78
|
-
# Notify the current queue size.
|
79
|
-
::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
|
85
|
+
supervisor_task.execute
|
80
86
|
end
|
81
|
-
supervisor_task.execute
|
82
87
|
end
|
83
|
-
end
|
84
88
|
|
89
|
+
def consumer_sampled_queue_size
|
90
|
+
consumers.values.map(&:sampled_queue_size).max
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
85
94
|
end
|
86
95
|
end
|
87
96
|
end
|
@@ -151,7 +151,9 @@ module ActivePublisher
|
|
151
151
|
|
152
152
|
def wait_for_confirms
|
153
153
|
return true unless channel.using_publisher_confirms?
|
154
|
-
|
154
|
+
::ActiveSupport::Notifications.instrument "publishes_confirmed.active_publisher" do
|
155
|
+
channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
|
156
|
+
end
|
155
157
|
end
|
156
158
|
end
|
157
159
|
end
|
@@ -9,28 +9,33 @@ module ActivePublisher
|
|
9
9
|
:timeout_interval => 5, # seconds
|
10
10
|
}
|
11
11
|
|
12
|
-
attr_reader :
|
12
|
+
attr_reader :consumers, :queue, :supervisor
|
13
13
|
|
14
14
|
def initialize(redis_pool)
|
15
15
|
@queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
|
16
|
-
|
16
|
+
@consumers = {}
|
17
|
+
create_and_supervise_consumers!
|
17
18
|
end
|
18
19
|
|
19
|
-
def
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
consumer
|
26
|
-
|
20
|
+
def create_and_supervise_consumers!
|
21
|
+
::ActivePublisher.configuration.publisher_threads.times do
|
22
|
+
consumer_id = ::SecureRandom.uuid
|
23
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
24
|
+
|
25
|
+
supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
|
26
|
+
consumer = consumers[consumer_id]
|
27
|
+
unless consumer.alive?
|
28
|
+
consumer.kill rescue nil
|
29
|
+
consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
|
30
|
+
::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
|
31
|
+
end
|
32
|
+
|
33
|
+
# Notify the current queue size.
|
34
|
+
::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
|
27
35
|
end
|
28
36
|
|
29
|
-
|
30
|
-
::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
|
37
|
+
supervisor_task.execute
|
31
38
|
end
|
32
|
-
|
33
|
-
supervisor_task.execute
|
34
39
|
end
|
35
40
|
|
36
41
|
def size
|
@@ -12,6 +12,7 @@ module ActivePublisher
|
|
12
12
|
:network_recovery_interval,
|
13
13
|
:password,
|
14
14
|
:port,
|
15
|
+
:publisher_threads,
|
15
16
|
:publisher_confirms,
|
16
17
|
:publisher_confirms_timeout,
|
17
18
|
:seconds_to_wait_for_graceful_shutdown,
|
@@ -41,6 +42,7 @@ module ActivePublisher
|
|
41
42
|
:max_async_publisher_lag_time => 10,
|
42
43
|
:network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
|
43
44
|
:port => 5672,
|
45
|
+
:publisher_threads => 1,
|
44
46
|
:publisher_confirms => false,
|
45
47
|
:publisher_confirms_timeout => 5_000, #specified as a number of milliseconds
|
46
48
|
:seconds_to_wait_for_graceful_shutdown => 30,
|
data/lib/active_publisher.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_publisher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Stien
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: exe
|
14
14
|
cert_chain: []
|
15
|
-
date: 2021-
|
15
|
+
date: 2021-12-02 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: bunny
|
@@ -223,8 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
223
223
|
- !ruby/object:Gem::Version
|
224
224
|
version: '0'
|
225
225
|
requirements: []
|
226
|
-
|
227
|
-
rubygems_version: 2.7.6
|
226
|
+
rubygems_version: 3.0.1
|
228
227
|
signing_key:
|
229
228
|
specification_version: 4
|
230
229
|
summary: Aims to make publishing work across MRI and jRuby painless and add some nice
|