active_publisher 1.2.2 → 1.2.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +56 -0
- data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb +79 -49
- data/lib/active_publisher/async/redis_adapter.rb +5 -3
- data/lib/active_publisher/configuration.rb +13 -2
- data/lib/active_publisher/version.rb +1 -1
- metadata +7 -7
- data/.travis.yml +0 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 736d696175390213664c2c01397673dff014a7aea213b64c99851a3d962aa3d2
|
4
|
+
data.tar.gz: 2f7efd868a202eec1513322d638023b00aa16c5e3f74c14a919484c06cb75289
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a2e4c032c91c55dcffd522da05251167007aca709ccfb62f3bd4bff9b20957426109c92bde1b526a40f10996ae93c07628a9d00b31b0ebaf220e3496a0404cf3
|
7
|
+
data.tar.gz: 52fcbde823b44976d7a8478ebb13d420fb9bd261675d9462372e8edc11cd01aa7b8705ccd39c44a278c9bdafc48fd9a546495a577ed9c5c9f85d61b6b51ae18e
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# Inspired by: http://mikebian.co/running-tests-against-multiple-ruby-versions-using-circleci/
|
2
|
+
|
3
|
+
version: 2.1
|
4
|
+
|
5
|
+
orbs:
|
6
|
+
ruby: circleci/ruby@1.1
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
test:
|
10
|
+
parallelism: 1
|
11
|
+
parameters:
|
12
|
+
ruby-image:
|
13
|
+
type: string
|
14
|
+
docker:
|
15
|
+
- image: << parameters.ruby-image >>
|
16
|
+
- image: rabbitmq
|
17
|
+
|
18
|
+
steps:
|
19
|
+
- checkout
|
20
|
+
- run:
|
21
|
+
name: install dockerize
|
22
|
+
command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
|
23
|
+
environment:
|
24
|
+
DOCKERIZE_VERSION: v0.3.0
|
25
|
+
- run:
|
26
|
+
name: Wait for rabbitmq
|
27
|
+
command: dockerize -wait tcp://localhost:5672 -timeout 1m
|
28
|
+
- run:
|
29
|
+
name: Install bundler
|
30
|
+
command: gem install bundler
|
31
|
+
- run:
|
32
|
+
name: Which bundler?
|
33
|
+
command: bundle -v
|
34
|
+
- run:
|
35
|
+
name: bundle install
|
36
|
+
command: bundle install
|
37
|
+
- run:
|
38
|
+
name: rspec
|
39
|
+
command: bundle exec rspec
|
40
|
+
|
41
|
+
# strangely, there seems to be very little documentation about exactly how martix builds work.
|
42
|
+
# By defining a param inside your job definition, Circle CI will automatically spawn a job for
|
43
|
+
# unique param value passed via `matrix`. Neat!
|
44
|
+
# https://circleci.com/blog/circleci-matrix-jobs/
|
45
|
+
workflows:
|
46
|
+
build_and_test:
|
47
|
+
jobs:
|
48
|
+
- test:
|
49
|
+
matrix:
|
50
|
+
parameters:
|
51
|
+
ruby-image:
|
52
|
+
- circleci/ruby:2.5
|
53
|
+
- circleci/ruby:2.6
|
54
|
+
- circleci/ruby:2.7
|
55
|
+
- circleci/jruby:9.1
|
56
|
+
- circleci/jruby:9.2
|
@@ -2,7 +2,13 @@ module ActivePublisher
|
|
2
2
|
module Async
|
3
3
|
module InMemoryAdapter
|
4
4
|
class ConsumerThread
|
5
|
-
attr_reader :thread, :queue, :sampled_queue_size, :last_tick_at
|
5
|
+
attr_reader :channel, :flush_max, :thread, :queue, :sampled_queue_size, :last_tick_at
|
6
|
+
|
7
|
+
if ::RUBY_PLATFORM == "java"
|
8
|
+
CHANNEL_CLOSED_ERRORS = [::MarchHare::ChannelAlreadyClosed]
|
9
|
+
else
|
10
|
+
CHANNEL_CLOSED_ERRORS = [::Bunny::ChannelAlreadyClosed]
|
11
|
+
end
|
6
12
|
|
7
13
|
if ::RUBY_PLATFORM == "java"
|
8
14
|
NETWORK_ERRORS = [::MarchHare::NetworkException, ::MarchHare::ConnectionRefused,
|
@@ -21,6 +27,7 @@ module ActivePublisher
|
|
21
27
|
def initialize(listen_queue)
|
22
28
|
@queue = listen_queue
|
23
29
|
@sampled_queue_size = queue.size
|
30
|
+
@flush_max = ::ActivePublisher.configuration.messages_per_batch
|
24
31
|
|
25
32
|
update_last_tick_at
|
26
33
|
start_thread
|
@@ -45,6 +52,32 @@ module ActivePublisher
|
|
45
52
|
end
|
46
53
|
end
|
47
54
|
|
55
|
+
def cleanup_up_channel
|
56
|
+
return if channel.nil?
|
57
|
+
channel.close
|
58
|
+
rescue => error
|
59
|
+
::ActivePublisher.configuration.error_handler.call(error, {:status => "Cleaning up the channel"})
|
60
|
+
end
|
61
|
+
|
62
|
+
def handle_current_messages_on_unknown_error(current_messages)
|
63
|
+
current_messages.each do |message|
|
64
|
+
# Degrade to single message publish ... or at least attempt to
|
65
|
+
begin
|
66
|
+
::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
|
67
|
+
current_messages.delete(message)
|
68
|
+
rescue *CHANNEL_CLOSED_ERRORS
|
69
|
+
# If the channel is bad, raise!
|
70
|
+
raise
|
71
|
+
rescue *PRECONDITION_ERRORS => error
|
72
|
+
# Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
|
73
|
+
::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
|
74
|
+
current_messages.delete(message)
|
75
|
+
rescue => other_error
|
76
|
+
::ActivePublisher.configuration.error_handler.call(other_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
48
81
|
def make_channel
|
49
82
|
channel = ::ActivePublisher::Async::InMemoryAdapter::Channel.new
|
50
83
|
channel.confirm_select if ::ActivePublisher.configuration.publisher_confirms
|
@@ -57,57 +90,54 @@ module ActivePublisher
|
|
57
90
|
|
58
91
|
def start_thread
|
59
92
|
return if alive?
|
60
|
-
@thread = ::Thread.new
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
await_network_reconnect
|
83
|
-
rescue => unknown_error
|
84
|
-
::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
|
85
|
-
current_messages.each do |message|
|
86
|
-
# Degrade to single message publish ... or at least attempt to
|
87
|
-
begin
|
88
|
-
::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
|
89
|
-
current_messages.delete(message)
|
90
|
-
rescue *PRECONDITION_ERRORS => error
|
91
|
-
# Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
|
92
|
-
::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
|
93
|
-
current_messages.delete(message)
|
94
|
-
rescue => individual_error
|
95
|
-
::ActivePublisher.configuration.error_handler.call(individual_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
# TODO: Find a way to bubble this out of the thread for logging purposes.
|
100
|
-
# Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
|
101
|
-
raise unknown_error
|
102
|
-
ensure
|
103
|
-
# Always requeue anything that gets stuck.
|
104
|
-
queue.concat(current_messages) if current_messages && !current_messages.empty?
|
93
|
+
@thread = ::Thread.new { start_consuming_thread }
|
94
|
+
end
|
95
|
+
|
96
|
+
def start_consuming_thread
|
97
|
+
loop do
|
98
|
+
# Sample the queue size so we don't shutdown when messages are in flight.
|
99
|
+
@sampled_queue_size = queue.size
|
100
|
+
current_messages = queue.pop_up_to(flush_max, :timeout => 0.1)
|
101
|
+
update_last_tick_at
|
102
|
+
# If the queue is empty, we should continue to update to "last_tick_at" time.
|
103
|
+
next if current_messages.nil?
|
104
|
+
|
105
|
+
@channel ||= make_channel
|
106
|
+
|
107
|
+
# We only look at active publisher messages. Everything else is dropped.
|
108
|
+
current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
|
109
|
+
|
110
|
+
begin
|
111
|
+
# Only open a single connection for each group of messages to an exchange
|
112
|
+
current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
|
113
|
+
publish_all(exchange_name, messages)
|
114
|
+
current_messages -= messages
|
105
115
|
end
|
116
|
+
rescue *CHANNEL_CLOSED_ERRORS
|
117
|
+
# If the channel is bad, raise without sending one-by-one!
|
118
|
+
raise
|
119
|
+
rescue *NETWORK_ERRORS
|
120
|
+
# Sleep because connection is down
|
121
|
+
await_network_reconnect
|
122
|
+
rescue => unknown_error
|
123
|
+
::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
|
124
|
+
|
125
|
+
# Attempt to deliver a message one-by-one. Raise if a closed channel error appears.
|
126
|
+
handle_current_messages_on_unknown_error(current_messages)
|
127
|
+
|
128
|
+
# TODO: Find a way to bubble this out of the thread for logging purposes.
|
129
|
+
# Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
|
130
|
+
raise unknown_error
|
131
|
+
ensure
|
132
|
+
# Always requeue anything that gets stuck.
|
133
|
+
queue.concat(current_messages) if current_messages && !current_messages.empty?
|
106
134
|
end
|
107
135
|
end
|
136
|
+
ensure
|
137
|
+
cleanup_up_channel
|
108
138
|
end
|
109
139
|
|
110
|
-
def publish_all(
|
140
|
+
def publish_all(exchange_name, messages)
|
111
141
|
exchange = channel.topic(exchange_name)
|
112
142
|
messages.each do |message|
|
113
143
|
fail ::ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
|
@@ -116,10 +146,10 @@ module ActivePublisher
|
|
116
146
|
exchange.publish(message.payload, options)
|
117
147
|
end
|
118
148
|
end
|
119
|
-
wait_for_confirms
|
149
|
+
wait_for_confirms
|
120
150
|
end
|
121
151
|
|
122
|
-
def wait_for_confirms
|
152
|
+
def wait_for_confirms
|
123
153
|
return true unless channel.using_publisher_confirms?
|
124
154
|
channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
|
125
155
|
end
|
@@ -19,7 +19,7 @@ module ActivePublisher
|
|
19
19
|
}
|
20
20
|
include ::ActivePublisher::Logging
|
21
21
|
|
22
|
-
attr_reader :async_queue, :redis_pool, :queue
|
22
|
+
attr_reader :async_queue, :flush_max, :flush_min, :redis_pool, :queue
|
23
23
|
|
24
24
|
def initialize(new_redis_pool)
|
25
25
|
logger.info "Starting redis publisher adapter"
|
@@ -27,6 +27,8 @@ module ActivePublisher
|
|
27
27
|
@redis_pool = new_redis_pool
|
28
28
|
@async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
|
29
29
|
@queue = ::MultiOpQueue::Queue.new
|
30
|
+
@flush_max = ::ActivePublisher.configuration.messages_per_batch
|
31
|
+
@flush_min = @flush_max / 2
|
30
32
|
|
31
33
|
supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
|
32
34
|
queue_size = queue.size
|
@@ -41,7 +43,7 @@ module ActivePublisher
|
|
41
43
|
def publish(route, payload, exchange_name, options = {})
|
42
44
|
message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
|
43
45
|
queue << ::Marshal.dump(message)
|
44
|
-
flush_queue! if queue.size >=
|
46
|
+
flush_queue! if queue.size >= flush_min || options[:flush_queue]
|
45
47
|
|
46
48
|
nil
|
47
49
|
end
|
@@ -58,7 +60,7 @@ module ActivePublisher
|
|
58
60
|
|
59
61
|
def flush_queue!
|
60
62
|
return if queue.empty?
|
61
|
-
encoded_messages = queue.pop_up_to(
|
63
|
+
encoded_messages = queue.pop_up_to(flush_max, :timeout => 0.001)
|
62
64
|
|
63
65
|
return if encoded_messages.nil?
|
64
66
|
return unless encoded_messages.respond_to?(:each)
|
@@ -8,6 +8,7 @@ module ActivePublisher
|
|
8
8
|
:host,
|
9
9
|
:hosts,
|
10
10
|
:max_async_publisher_lag_time,
|
11
|
+
:messages_per_batch,
|
11
12
|
:network_recovery_interval,
|
12
13
|
:password,
|
13
14
|
:port,
|
@@ -36,6 +37,7 @@ module ActivePublisher
|
|
36
37
|
:host => "localhost",
|
37
38
|
:hosts => [],
|
38
39
|
:password => "guest",
|
40
|
+
:messages_per_batch => 25,
|
39
41
|
:max_async_publisher_lag_time => 10,
|
40
42
|
:network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
|
41
43
|
:port => 5672,
|
@@ -63,8 +65,8 @@ module ActivePublisher
|
|
63
65
|
|
64
66
|
yaml_config = attempt_to_load_yaml_file(env)
|
65
67
|
DEFAULTS.each_pair do |key, value|
|
66
|
-
setting =
|
67
|
-
::ActivePublisher.configuration.public_send("#{key}=", setting) if
|
68
|
+
exists, setting = fetch_config_value(key, cli_options, yaml_config)
|
69
|
+
::ActivePublisher.configuration.public_send("#{key}=", setting) if exists
|
68
70
|
end
|
69
71
|
|
70
72
|
true
|
@@ -88,6 +90,15 @@ module ActivePublisher
|
|
88
90
|
end
|
89
91
|
private_class_method :attempt_to_load_yaml_file
|
90
92
|
|
93
|
+
def self.fetch_config_value(key, cli_options, yaml_config)
|
94
|
+
return [true, cli_options[key]] if cli_options.key?(key)
|
95
|
+
return [true, cli_options[key.to_s]] if cli_options.key?(key.to_s)
|
96
|
+
return [true, yaml_config[key]] if yaml_config.key?(key)
|
97
|
+
return [true, yaml_config[key.to_s]] if yaml_config.key?(key.to_s)
|
98
|
+
[false, nil]
|
99
|
+
end
|
100
|
+
private_class_method :fetch_config_value
|
101
|
+
|
91
102
|
##
|
92
103
|
# Instance Methods
|
93
104
|
#
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_publisher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Stien
|
@@ -9,10 +9,10 @@ authors:
|
|
9
9
|
- Brandon Dewitt
|
10
10
|
- Devin Christensen
|
11
11
|
- Michael Ries
|
12
|
-
autorequire:
|
12
|
+
autorequire:
|
13
13
|
bindir: exe
|
14
14
|
cert_chain: []
|
15
|
-
date:
|
15
|
+
date: 2021-09-29 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: bunny
|
@@ -179,9 +179,9 @@ executables: []
|
|
179
179
|
extensions: []
|
180
180
|
extra_rdoc_files: []
|
181
181
|
files:
|
182
|
+
- ".circleci/config.yml"
|
182
183
|
- ".gitignore"
|
183
184
|
- ".rspec"
|
184
|
-
- ".travis.yml"
|
185
185
|
- CODE_OF_CONDUCT.md
|
186
186
|
- Gemfile
|
187
187
|
- LICENSE.txt
|
@@ -208,7 +208,7 @@ homepage: https://github.com/mxenabled/active_publisher
|
|
208
208
|
licenses:
|
209
209
|
- MIT
|
210
210
|
metadata: {}
|
211
|
-
post_install_message:
|
211
|
+
post_install_message:
|
212
212
|
rdoc_options: []
|
213
213
|
require_paths:
|
214
214
|
- lib
|
@@ -223,9 +223,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
223
223
|
- !ruby/object:Gem::Version
|
224
224
|
version: '0'
|
225
225
|
requirements: []
|
226
|
-
rubyforge_project:
|
226
|
+
rubyforge_project:
|
227
227
|
rubygems_version: 2.7.6
|
228
|
-
signing_key:
|
228
|
+
signing_key:
|
229
229
|
specification_version: 4
|
230
230
|
summary: Aims to make publishing work across MRI and jRuby painless and add some nice
|
231
231
|
features like automatially publishing lifecycle events for ActiveRecord models.
|