active_publisher 1.2.2-java → 1.2.6-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8b9c83ffe7b068d340c60ce3487b409eff93776ef7e1a431f7493aede45c7098
4
- data.tar.gz: 9a5019a2d7be9a60ca2afa5e1b60b2dd122c599da029f5908c0933116801dd0f
3
+ metadata.gz: 5eb0ac3e4acd1a5a7cd668d9eef3bd900af1964781b65eb22c3b97d17add28b4
4
+ data.tar.gz: 8bb17f982325ef1d4923939f4e13729affddfc4320c793305837d9a368c51f22
5
5
  SHA512:
6
- metadata.gz: 586bca9328785ab472ce142fa2224dd20c16a3d2ddc1105b5dc8a98f67ed5d6fe23751074991a16ec6162327512373220bf11a84577eed62bc0027e8aa62c460
7
- data.tar.gz: 7d4bfc3de040b8d27232f01c89794976667f3e607a30f01f6c99a747de6f42477168d775fdd749d797b15af9ad6814a6338ed3f98a299ba0c6b8a137773a8490
6
+ metadata.gz: 8ce688832579a6ac3eb94051b43b5fd956c10bf8c1e64b1a7354c806543674810a6784f18caab97165b0115cc52e9aa156732f04cd2be93bd69e36bf38eb9028
7
+ data.tar.gz: 0fd321640fe284276f11c5e262df01f9731e1ea9a870cddc22a85fee6753663ee315fc427c2d1aa0ecab29afc35c31e9cefbd14283db4491cb16fb11398ddf3e
@@ -0,0 +1,56 @@
1
+ # Inspired by: http://mikebian.co/running-tests-against-multiple-ruby-versions-using-circleci/
2
+
3
+ version: 2.1
4
+
5
+ orbs:
6
+ ruby: circleci/ruby@1.1
7
+
8
+ jobs:
9
+ test:
10
+ parallelism: 1
11
+ parameters:
12
+ ruby-image:
13
+ type: string
14
+ docker:
15
+ - image: << parameters.ruby-image >>
16
+ - image: rabbitmq
17
+
18
+ steps:
19
+ - checkout
20
+ - run:
21
+ name: install dockerize
22
+ command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
23
+ environment:
24
+ DOCKERIZE_VERSION: v0.3.0
25
+ - run:
26
+ name: Wait for rabbitmq
27
+ command: dockerize -wait tcp://localhost:5672 -timeout 1m
28
+ - run:
29
+ name: Install bundler
30
+ command: gem install bundler
31
+ - run:
32
+ name: Which bundler?
33
+ command: bundle -v
34
+ - run:
35
+ name: bundle install
36
+ command: bundle install
37
+ - run:
38
+ name: rspec
39
+ command: bundle exec rspec
40
+
41
+ # strangely, there seems to be very little documentation about exactly how martix builds work.
42
+ # By defining a param inside your job definition, Circle CI will automatically spawn a job for
43
+ # unique param value passed via `matrix`. Neat!
44
+ # https://circleci.com/blog/circleci-matrix-jobs/
45
+ workflows:
46
+ build_and_test:
47
+ jobs:
48
+ - test:
49
+ matrix:
50
+ parameters:
51
+ ruby-image:
52
+ - circleci/ruby:2.5
53
+ - circleci/ruby:2.6
54
+ - circleci/ruby:2.7
55
+ - circleci/jruby:9.1
56
+ - circleci/jruby:9.2
@@ -2,7 +2,13 @@ module ActivePublisher
2
2
  module Async
3
3
  module InMemoryAdapter
4
4
  class ConsumerThread
5
- attr_reader :thread, :queue, :sampled_queue_size, :last_tick_at
5
+ attr_reader :channel, :flush_max, :thread, :queue, :sampled_queue_size, :last_tick_at
6
+
7
+ if ::RUBY_PLATFORM == "java"
8
+ CHANNEL_CLOSED_ERRORS = [::MarchHare::ChannelAlreadyClosed]
9
+ else
10
+ CHANNEL_CLOSED_ERRORS = [::Bunny::ChannelAlreadyClosed]
11
+ end
6
12
 
7
13
  if ::RUBY_PLATFORM == "java"
8
14
  NETWORK_ERRORS = [::MarchHare::NetworkException, ::MarchHare::ConnectionRefused,
@@ -21,6 +27,7 @@ module ActivePublisher
21
27
  def initialize(listen_queue)
22
28
  @queue = listen_queue
23
29
  @sampled_queue_size = queue.size
30
+ @flush_max = ::ActivePublisher.configuration.messages_per_batch
24
31
 
25
32
  update_last_tick_at
26
33
  start_thread
@@ -45,6 +52,32 @@ module ActivePublisher
45
52
  end
46
53
  end
47
54
 
55
+ def cleanup_up_channel
56
+ return if channel.nil?
57
+ channel.close
58
+ rescue => error
59
+ ::ActivePublisher.configuration.error_handler.call(error, {:status => "Cleaning up the channel"})
60
+ end
61
+
62
+ def handle_current_messages_on_unknown_error(current_messages)
63
+ current_messages.each do |message|
64
+ # Degrade to single message publish ... or at least attempt to
65
+ begin
66
+ ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
67
+ current_messages.delete(message)
68
+ rescue *CHANNEL_CLOSED_ERRORS
69
+ # If the channel is bad, raise!
70
+ raise
71
+ rescue *PRECONDITION_ERRORS => error
72
+ # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
73
+ ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
74
+ current_messages.delete(message)
75
+ rescue => other_error
76
+ ::ActivePublisher.configuration.error_handler.call(other_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
77
+ end
78
+ end
79
+ end
80
+
48
81
  def make_channel
49
82
  channel = ::ActivePublisher::Async::InMemoryAdapter::Channel.new
50
83
  channel.confirm_select if ::ActivePublisher.configuration.publisher_confirms
@@ -57,57 +90,54 @@ module ActivePublisher
57
90
 
58
91
  def start_thread
59
92
  return if alive?
60
- @thread = ::Thread.new do
61
- loop do
62
- # Sample the queue size so we don't shutdown when messages are in flight.
63
- @sampled_queue_size = queue.size
64
- current_messages = queue.pop_up_to(50, :timeout => 0.1)
65
- update_last_tick_at
66
- # If the queue is empty, we should continue to update to "last_tick_at" time.
67
- next if current_messages.nil?
68
-
69
- # We only look at active publisher messages. Everything else is dropped.
70
- current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
71
-
72
- begin
73
- @channel ||= make_channel
74
-
75
- # Only open a single connection for each group of messages to an exchange
76
- current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
77
- publish_all(@channel, exchange_name, messages)
78
- current_messages -= messages
79
- end
80
- rescue *NETWORK_ERRORS
81
- # Sleep because connection is down
82
- await_network_reconnect
83
- rescue => unknown_error
84
- ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
85
- current_messages.each do |message|
86
- # Degrade to single message publish ... or at least attempt to
87
- begin
88
- ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
89
- current_messages.delete(message)
90
- rescue *PRECONDITION_ERRORS => error
91
- # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
92
- ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
93
- current_messages.delete(message)
94
- rescue => individual_error
95
- ::ActivePublisher.configuration.error_handler.call(individual_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
96
- end
97
- end
98
-
99
- # TODO: Find a way to bubble this out of the thread for logging purposes.
100
- # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
101
- raise unknown_error
102
- ensure
103
- # Always requeue anything that gets stuck.
104
- queue.concat(current_messages) if current_messages && !current_messages.empty?
93
+ @thread = ::Thread.new { start_consuming_thread }
94
+ end
95
+
96
+ def start_consuming_thread
97
+ loop do
98
+ # Sample the queue size so we don't shutdown when messages are in flight.
99
+ @sampled_queue_size = queue.size
100
+ current_messages = queue.pop_up_to(flush_max, :timeout => 0.1)
101
+ update_last_tick_at
102
+ # If the queue is empty, we should continue to update to "last_tick_at" time.
103
+ next if current_messages.nil?
104
+
105
+ @channel ||= make_channel
106
+
107
+ # We only look at active publisher messages. Everything else is dropped.
108
+ current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
109
+
110
+ begin
111
+ # Only open a single connection for each group of messages to an exchange
112
+ current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
113
+ publish_all(exchange_name, messages)
114
+ current_messages -= messages
105
115
  end
116
+ rescue *CHANNEL_CLOSED_ERRORS
117
+ # If the channel is bad, raise without sending one-by-one!
118
+ raise
119
+ rescue *NETWORK_ERRORS
120
+ # Sleep because connection is down
121
+ await_network_reconnect
122
+ rescue => unknown_error
123
+ ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
124
+
125
+ # Attempt to deliver a message one-by-one. Raise if a closed channel error appears.
126
+ handle_current_messages_on_unknown_error(current_messages)
127
+
128
+ # TODO: Find a way to bubble this out of the thread for logging purposes.
129
+ # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
130
+ raise unknown_error
131
+ ensure
132
+ # Always requeue anything that gets stuck.
133
+ queue.concat(current_messages) if current_messages && !current_messages.empty?
106
134
  end
107
135
  end
136
+ ensure
137
+ cleanup_up_channel
108
138
  end
109
139
 
110
- def publish_all(channel, exchange_name, messages)
140
+ def publish_all(exchange_name, messages)
111
141
  exchange = channel.topic(exchange_name)
112
142
  messages.each do |message|
113
143
  fail ::ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
@@ -116,10 +146,10 @@ module ActivePublisher
116
146
  exchange.publish(message.payload, options)
117
147
  end
118
148
  end
119
- wait_for_confirms(channel)
149
+ wait_for_confirms
120
150
  end
121
151
 
122
- def wait_for_confirms(channel)
152
+ def wait_for_confirms
123
153
  return true unless channel.using_publisher_confirms?
124
154
  channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
125
155
  end
@@ -19,7 +19,7 @@ module ActivePublisher
19
19
  }
20
20
  include ::ActivePublisher::Logging
21
21
 
22
- attr_reader :async_queue, :redis_pool, :queue
22
+ attr_reader :async_queue, :flush_max, :flush_min, :redis_pool, :queue
23
23
 
24
24
  def initialize(new_redis_pool)
25
25
  logger.info "Starting redis publisher adapter"
@@ -27,6 +27,8 @@ module ActivePublisher
27
27
  @redis_pool = new_redis_pool
28
28
  @async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
29
29
  @queue = ::MultiOpQueue::Queue.new
30
+ @flush_max = ::ActivePublisher.configuration.messages_per_batch
31
+ @flush_min = @flush_max / 2
30
32
 
31
33
  supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
32
34
  queue_size = queue.size
@@ -41,7 +43,7 @@ module ActivePublisher
41
43
  def publish(route, payload, exchange_name, options = {})
42
44
  message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
43
45
  queue << ::Marshal.dump(message)
44
- flush_queue! if queue.size >= 20 || options[:flush_queue]
46
+ flush_queue! if queue.size >= flush_min || options[:flush_queue]
45
47
 
46
48
  nil
47
49
  end
@@ -58,7 +60,7 @@ module ActivePublisher
58
60
 
59
61
  def flush_queue!
60
62
  return if queue.empty?
61
- encoded_messages = queue.pop_up_to(25, :timeout => 0.001)
63
+ encoded_messages = queue.pop_up_to(flush_max, :timeout => 0.001)
62
64
 
63
65
  return if encoded_messages.nil?
64
66
  return unless encoded_messages.respond_to?(:each)
@@ -8,6 +8,7 @@ module ActivePublisher
8
8
  :host,
9
9
  :hosts,
10
10
  :max_async_publisher_lag_time,
11
+ :messages_per_batch,
11
12
  :network_recovery_interval,
12
13
  :password,
13
14
  :port,
@@ -36,6 +37,7 @@ module ActivePublisher
36
37
  :host => "localhost",
37
38
  :hosts => [],
38
39
  :password => "guest",
40
+ :messages_per_batch => 25,
39
41
  :max_async_publisher_lag_time => 10,
40
42
  :network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
41
43
  :port => 5672,
@@ -63,8 +65,8 @@ module ActivePublisher
63
65
 
64
66
  yaml_config = attempt_to_load_yaml_file(env)
65
67
  DEFAULTS.each_pair do |key, value|
66
- setting = cli_options[key] || cli_options[key.to_s] || yaml_config[key] || yaml_config[key.to_s]
67
- ::ActivePublisher.configuration.public_send("#{key}=", setting) if setting
68
+ exists, setting = fetch_config_value(key, cli_options, yaml_config)
69
+ ::ActivePublisher.configuration.public_send("#{key}=", setting) if exists
68
70
  end
69
71
 
70
72
  true
@@ -88,6 +90,15 @@ module ActivePublisher
88
90
  end
89
91
  private_class_method :attempt_to_load_yaml_file
90
92
 
93
+ def self.fetch_config_value(key, cli_options, yaml_config)
94
+ return [true, cli_options[key]] if cli_options.key?(key)
95
+ return [true, cli_options[key.to_s]] if cli_options.key?(key.to_s)
96
+ return [true, yaml_config[key]] if yaml_config.key?(key)
97
+ return [true, yaml_config[key.to_s]] if yaml_config.key?(key.to_s)
98
+ [false, nil]
99
+ end
100
+ private_class_method :fetch_config_value
101
+
91
102
  ##
92
103
  # Instance Methods
93
104
  #
@@ -1,3 +1,3 @@
1
1
  module ActivePublisher
2
- VERSION = "1.2.2"
2
+ VERSION = "1.2.6"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: active_publisher
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.2
4
+ version: 1.2.6
5
5
  platform: java
6
6
  authors:
7
7
  - Brian Stien
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: exe
14
14
  cert_chain: []
15
- date: 2020-05-19 00:00:00.000000000 Z
15
+ date: 2021-09-29 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  requirement: !ruby/object:Gem::Requirement
@@ -21,8 +21,8 @@ dependencies:
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2.7'
23
23
  name: march_hare
24
- prerelease: false
25
24
  type: :runtime
25
+ prerelease: false
26
26
  version_requirements: !ruby/object:Gem::Requirement
27
27
  requirements:
28
28
  - - "~>"
@@ -35,8 +35,8 @@ dependencies:
35
35
  - !ruby/object:Gem::Version
36
36
  version: '3.2'
37
37
  name: activesupport
38
- prerelease: false
39
38
  type: :runtime
39
+ prerelease: false
40
40
  version_requirements: !ruby/object:Gem::Requirement
41
41
  requirements:
42
42
  - - ">="
@@ -49,8 +49,8 @@ dependencies:
49
49
  - !ruby/object:Gem::Version
50
50
  version: '0'
51
51
  name: concurrent-ruby
52
- prerelease: false
53
52
  type: :runtime
53
+ prerelease: false
54
54
  version_requirements: !ruby/object:Gem::Requirement
55
55
  requirements:
56
56
  - - ">="
@@ -63,8 +63,8 @@ dependencies:
63
63
  - !ruby/object:Gem::Version
64
64
  version: 0.2.0
65
65
  name: multi_op_queue
66
- prerelease: false
67
66
  type: :runtime
67
+ prerelease: false
68
68
  version_requirements: !ruby/object:Gem::Requirement
69
69
  requirements:
70
70
  - - ">="
@@ -77,8 +77,8 @@ dependencies:
77
77
  - !ruby/object:Gem::Version
78
78
  version: '0'
79
79
  name: benchmark-ips
80
- prerelease: false
81
80
  type: :development
81
+ prerelease: false
82
82
  version_requirements: !ruby/object:Gem::Requirement
83
83
  requirements:
84
84
  - - ">="
@@ -91,8 +91,8 @@ dependencies:
91
91
  - !ruby/object:Gem::Version
92
92
  version: '0'
93
93
  name: bundler
94
- prerelease: false
95
94
  type: :development
95
+ prerelease: false
96
96
  version_requirements: !ruby/object:Gem::Requirement
97
97
  requirements:
98
98
  - - ">="
@@ -105,8 +105,8 @@ dependencies:
105
105
  - !ruby/object:Gem::Version
106
106
  version: '0'
107
107
  name: connection_pool
108
- prerelease: false
109
108
  type: :development
109
+ prerelease: false
110
110
  version_requirements: !ruby/object:Gem::Requirement
111
111
  requirements:
112
112
  - - ">="
@@ -119,8 +119,8 @@ dependencies:
119
119
  - !ruby/object:Gem::Version
120
120
  version: '0'
121
121
  name: fakeredis
122
- prerelease: false
123
122
  type: :development
123
+ prerelease: false
124
124
  version_requirements: !ruby/object:Gem::Requirement
125
125
  requirements:
126
126
  - - ">="
@@ -133,8 +133,8 @@ dependencies:
133
133
  - !ruby/object:Gem::Version
134
134
  version: '0'
135
135
  name: pry
136
- prerelease: false
137
136
  type: :development
137
+ prerelease: false
138
138
  version_requirements: !ruby/object:Gem::Requirement
139
139
  requirements:
140
140
  - - ">="
@@ -147,8 +147,8 @@ dependencies:
147
147
  - !ruby/object:Gem::Version
148
148
  version: '0'
149
149
  name: rake
150
- prerelease: false
151
150
  type: :development
151
+ prerelease: false
152
152
  version_requirements: !ruby/object:Gem::Requirement
153
153
  requirements:
154
154
  - - ">="
@@ -161,8 +161,8 @@ dependencies:
161
161
  - !ruby/object:Gem::Version
162
162
  version: '3.2'
163
163
  name: rspec
164
- prerelease: false
165
164
  type: :development
165
+ prerelease: false
166
166
  version_requirements: !ruby/object:Gem::Requirement
167
167
  requirements:
168
168
  - - "~>"
@@ -179,9 +179,9 @@ executables: []
179
179
  extensions: []
180
180
  extra_rdoc_files: []
181
181
  files:
182
+ - ".circleci/config.yml"
182
183
  - ".gitignore"
183
184
  - ".rspec"
184
- - ".travis.yml"
185
185
  - CODE_OF_CONDUCT.md
186
186
  - Gemfile
187
187
  - LICENSE.txt
@@ -223,8 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
223
223
  - !ruby/object:Gem::Version
224
224
  version: '0'
225
225
  requirements: []
226
- rubyforge_project:
227
- rubygems_version: 2.7.9
226
+ rubygems_version: 3.0.6
228
227
  signing_key:
229
228
  specification_version: 4
230
229
  summary: Aims to make publishing work across MRI and jRuby painless and add some nice
data/.travis.yml DELETED
@@ -1,14 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.3.8
4
- - 2.5.7
5
- - jruby-9.1.12.0
6
- - jruby-9.2.7.0
7
- addons:
8
- apt:
9
- packages:
10
- - rabbitmq-server
11
- services:
12
- - rabbitmq
13
- sudo: false
14
- cache: bundler