active_publisher 1.2.2-java → 1.2.6-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8b9c83ffe7b068d340c60ce3487b409eff93776ef7e1a431f7493aede45c7098
4
- data.tar.gz: 9a5019a2d7be9a60ca2afa5e1b60b2dd122c599da029f5908c0933116801dd0f
3
+ metadata.gz: 5eb0ac3e4acd1a5a7cd668d9eef3bd900af1964781b65eb22c3b97d17add28b4
4
+ data.tar.gz: 8bb17f982325ef1d4923939f4e13729affddfc4320c793305837d9a368c51f22
5
5
  SHA512:
6
- metadata.gz: 586bca9328785ab472ce142fa2224dd20c16a3d2ddc1105b5dc8a98f67ed5d6fe23751074991a16ec6162327512373220bf11a84577eed62bc0027e8aa62c460
7
- data.tar.gz: 7d4bfc3de040b8d27232f01c89794976667f3e607a30f01f6c99a747de6f42477168d775fdd749d797b15af9ad6814a6338ed3f98a299ba0c6b8a137773a8490
6
+ metadata.gz: 8ce688832579a6ac3eb94051b43b5fd956c10bf8c1e64b1a7354c806543674810a6784f18caab97165b0115cc52e9aa156732f04cd2be93bd69e36bf38eb9028
7
+ data.tar.gz: 0fd321640fe284276f11c5e262df01f9731e1ea9a870cddc22a85fee6753663ee315fc427c2d1aa0ecab29afc35c31e9cefbd14283db4491cb16fb11398ddf3e
@@ -0,0 +1,56 @@
1
+ # Inspired by: http://mikebian.co/running-tests-against-multiple-ruby-versions-using-circleci/
2
+
3
+ version: 2.1
4
+
5
+ orbs:
6
+ ruby: circleci/ruby@1.1
7
+
8
+ jobs:
9
+ test:
10
+ parallelism: 1
11
+ parameters:
12
+ ruby-image:
13
+ type: string
14
+ docker:
15
+ - image: << parameters.ruby-image >>
16
+ - image: rabbitmq
17
+
18
+ steps:
19
+ - checkout
20
+ - run:
21
+ name: install dockerize
22
+ command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
23
+ environment:
24
+ DOCKERIZE_VERSION: v0.3.0
25
+ - run:
26
+ name: Wait for rabbitmq
27
+ command: dockerize -wait tcp://localhost:5672 -timeout 1m
28
+ - run:
29
+ name: Install bundler
30
+ command: gem install bundler
31
+ - run:
32
+ name: Which bundler?
33
+ command: bundle -v
34
+ - run:
35
+ name: bundle install
36
+ command: bundle install
37
+ - run:
38
+ name: rspec
39
+ command: bundle exec rspec
40
+
41
+ # strangely, there seems to be very little documentation about exactly how martix builds work.
42
+ # By defining a param inside your job definition, Circle CI will automatically spawn a job for
43
+ # unique param value passed via `matrix`. Neat!
44
+ # https://circleci.com/blog/circleci-matrix-jobs/
45
+ workflows:
46
+ build_and_test:
47
+ jobs:
48
+ - test:
49
+ matrix:
50
+ parameters:
51
+ ruby-image:
52
+ - circleci/ruby:2.5
53
+ - circleci/ruby:2.6
54
+ - circleci/ruby:2.7
55
+ - circleci/jruby:9.1
56
+ - circleci/jruby:9.2
@@ -2,7 +2,13 @@ module ActivePublisher
2
2
  module Async
3
3
  module InMemoryAdapter
4
4
  class ConsumerThread
5
- attr_reader :thread, :queue, :sampled_queue_size, :last_tick_at
5
+ attr_reader :channel, :flush_max, :thread, :queue, :sampled_queue_size, :last_tick_at
6
+
7
+ if ::RUBY_PLATFORM == "java"
8
+ CHANNEL_CLOSED_ERRORS = [::MarchHare::ChannelAlreadyClosed]
9
+ else
10
+ CHANNEL_CLOSED_ERRORS = [::Bunny::ChannelAlreadyClosed]
11
+ end
6
12
 
7
13
  if ::RUBY_PLATFORM == "java"
8
14
  NETWORK_ERRORS = [::MarchHare::NetworkException, ::MarchHare::ConnectionRefused,
@@ -21,6 +27,7 @@ module ActivePublisher
21
27
  def initialize(listen_queue)
22
28
  @queue = listen_queue
23
29
  @sampled_queue_size = queue.size
30
+ @flush_max = ::ActivePublisher.configuration.messages_per_batch
24
31
 
25
32
  update_last_tick_at
26
33
  start_thread
@@ -45,6 +52,32 @@ module ActivePublisher
45
52
  end
46
53
  end
47
54
 
55
+ def cleanup_up_channel
56
+ return if channel.nil?
57
+ channel.close
58
+ rescue => error
59
+ ::ActivePublisher.configuration.error_handler.call(error, {:status => "Cleaning up the channel"})
60
+ end
61
+
62
+ def handle_current_messages_on_unknown_error(current_messages)
63
+ current_messages.each do |message|
64
+ # Degrade to single message publish ... or at least attempt to
65
+ begin
66
+ ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
67
+ current_messages.delete(message)
68
+ rescue *CHANNEL_CLOSED_ERRORS
69
+ # If the channel is bad, raise!
70
+ raise
71
+ rescue *PRECONDITION_ERRORS => error
72
+ # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
73
+ ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
74
+ current_messages.delete(message)
75
+ rescue => other_error
76
+ ::ActivePublisher.configuration.error_handler.call(other_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
77
+ end
78
+ end
79
+ end
80
+
48
81
  def make_channel
49
82
  channel = ::ActivePublisher::Async::InMemoryAdapter::Channel.new
50
83
  channel.confirm_select if ::ActivePublisher.configuration.publisher_confirms
@@ -57,57 +90,54 @@ module ActivePublisher
57
90
 
58
91
  def start_thread
59
92
  return if alive?
60
- @thread = ::Thread.new do
61
- loop do
62
- # Sample the queue size so we don't shutdown when messages are in flight.
63
- @sampled_queue_size = queue.size
64
- current_messages = queue.pop_up_to(50, :timeout => 0.1)
65
- update_last_tick_at
66
- # If the queue is empty, we should continue to update to "last_tick_at" time.
67
- next if current_messages.nil?
68
-
69
- # We only look at active publisher messages. Everything else is dropped.
70
- current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
71
-
72
- begin
73
- @channel ||= make_channel
74
-
75
- # Only open a single connection for each group of messages to an exchange
76
- current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
77
- publish_all(@channel, exchange_name, messages)
78
- current_messages -= messages
79
- end
80
- rescue *NETWORK_ERRORS
81
- # Sleep because connection is down
82
- await_network_reconnect
83
- rescue => unknown_error
84
- ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
85
- current_messages.each do |message|
86
- # Degrade to single message publish ... or at least attempt to
87
- begin
88
- ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
89
- current_messages.delete(message)
90
- rescue *PRECONDITION_ERRORS => error
91
- # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
92
- ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
93
- current_messages.delete(message)
94
- rescue => individual_error
95
- ::ActivePublisher.configuration.error_handler.call(individual_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
96
- end
97
- end
98
-
99
- # TODO: Find a way to bubble this out of the thread for logging purposes.
100
- # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
101
- raise unknown_error
102
- ensure
103
- # Always requeue anything that gets stuck.
104
- queue.concat(current_messages) if current_messages && !current_messages.empty?
93
+ @thread = ::Thread.new { start_consuming_thread }
94
+ end
95
+
96
+ def start_consuming_thread
97
+ loop do
98
+ # Sample the queue size so we don't shutdown when messages are in flight.
99
+ @sampled_queue_size = queue.size
100
+ current_messages = queue.pop_up_to(flush_max, :timeout => 0.1)
101
+ update_last_tick_at
102
+ # If the queue is empty, we should continue to update to "last_tick_at" time.
103
+ next if current_messages.nil?
104
+
105
+ @channel ||= make_channel
106
+
107
+ # We only look at active publisher messages. Everything else is dropped.
108
+ current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
109
+
110
+ begin
111
+ # Only open a single connection for each group of messages to an exchange
112
+ current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
113
+ publish_all(exchange_name, messages)
114
+ current_messages -= messages
105
115
  end
116
+ rescue *CHANNEL_CLOSED_ERRORS
117
+ # If the channel is bad, raise without sending one-by-one!
118
+ raise
119
+ rescue *NETWORK_ERRORS
120
+ # Sleep because connection is down
121
+ await_network_reconnect
122
+ rescue => unknown_error
123
+ ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
124
+
125
+ # Attempt to deliver a message one-by-one. Raise if a closed channel error appears.
126
+ handle_current_messages_on_unknown_error(current_messages)
127
+
128
+ # TODO: Find a way to bubble this out of the thread for logging purposes.
129
+ # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
130
+ raise unknown_error
131
+ ensure
132
+ # Always requeue anything that gets stuck.
133
+ queue.concat(current_messages) if current_messages && !current_messages.empty?
106
134
  end
107
135
  end
136
+ ensure
137
+ cleanup_up_channel
108
138
  end
109
139
 
110
- def publish_all(channel, exchange_name, messages)
140
+ def publish_all(exchange_name, messages)
111
141
  exchange = channel.topic(exchange_name)
112
142
  messages.each do |message|
113
143
  fail ::ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
@@ -116,10 +146,10 @@ module ActivePublisher
116
146
  exchange.publish(message.payload, options)
117
147
  end
118
148
  end
119
- wait_for_confirms(channel)
149
+ wait_for_confirms
120
150
  end
121
151
 
122
- def wait_for_confirms(channel)
152
+ def wait_for_confirms
123
153
  return true unless channel.using_publisher_confirms?
124
154
  channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
125
155
  end
@@ -19,7 +19,7 @@ module ActivePublisher
19
19
  }
20
20
  include ::ActivePublisher::Logging
21
21
 
22
- attr_reader :async_queue, :redis_pool, :queue
22
+ attr_reader :async_queue, :flush_max, :flush_min, :redis_pool, :queue
23
23
 
24
24
  def initialize(new_redis_pool)
25
25
  logger.info "Starting redis publisher adapter"
@@ -27,6 +27,8 @@ module ActivePublisher
27
27
  @redis_pool = new_redis_pool
28
28
  @async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
29
29
  @queue = ::MultiOpQueue::Queue.new
30
+ @flush_max = ::ActivePublisher.configuration.messages_per_batch
31
+ @flush_min = @flush_max / 2
30
32
 
31
33
  supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
32
34
  queue_size = queue.size
@@ -41,7 +43,7 @@ module ActivePublisher
41
43
  def publish(route, payload, exchange_name, options = {})
42
44
  message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
43
45
  queue << ::Marshal.dump(message)
44
- flush_queue! if queue.size >= 20 || options[:flush_queue]
46
+ flush_queue! if queue.size >= flush_min || options[:flush_queue]
45
47
 
46
48
  nil
47
49
  end
@@ -58,7 +60,7 @@ module ActivePublisher
58
60
 
59
61
  def flush_queue!
60
62
  return if queue.empty?
61
- encoded_messages = queue.pop_up_to(25, :timeout => 0.001)
63
+ encoded_messages = queue.pop_up_to(flush_max, :timeout => 0.001)
62
64
 
63
65
  return if encoded_messages.nil?
64
66
  return unless encoded_messages.respond_to?(:each)
@@ -8,6 +8,7 @@ module ActivePublisher
8
8
  :host,
9
9
  :hosts,
10
10
  :max_async_publisher_lag_time,
11
+ :messages_per_batch,
11
12
  :network_recovery_interval,
12
13
  :password,
13
14
  :port,
@@ -36,6 +37,7 @@ module ActivePublisher
36
37
  :host => "localhost",
37
38
  :hosts => [],
38
39
  :password => "guest",
40
+ :messages_per_batch => 25,
39
41
  :max_async_publisher_lag_time => 10,
40
42
  :network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
41
43
  :port => 5672,
@@ -63,8 +65,8 @@ module ActivePublisher
63
65
 
64
66
  yaml_config = attempt_to_load_yaml_file(env)
65
67
  DEFAULTS.each_pair do |key, value|
66
- setting = cli_options[key] || cli_options[key.to_s] || yaml_config[key] || yaml_config[key.to_s]
67
- ::ActivePublisher.configuration.public_send("#{key}=", setting) if setting
68
+ exists, setting = fetch_config_value(key, cli_options, yaml_config)
69
+ ::ActivePublisher.configuration.public_send("#{key}=", setting) if exists
68
70
  end
69
71
 
70
72
  true
@@ -88,6 +90,15 @@ module ActivePublisher
88
90
  end
89
91
  private_class_method :attempt_to_load_yaml_file
90
92
 
93
+ def self.fetch_config_value(key, cli_options, yaml_config)
94
+ return [true, cli_options[key]] if cli_options.key?(key)
95
+ return [true, cli_options[key.to_s]] if cli_options.key?(key.to_s)
96
+ return [true, yaml_config[key]] if yaml_config.key?(key)
97
+ return [true, yaml_config[key.to_s]] if yaml_config.key?(key.to_s)
98
+ [false, nil]
99
+ end
100
+ private_class_method :fetch_config_value
101
+
91
102
  ##
92
103
  # Instance Methods
93
104
  #
@@ -1,3 +1,3 @@
1
1
  module ActivePublisher
2
- VERSION = "1.2.2"
2
+ VERSION = "1.2.6"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: active_publisher
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.2
4
+ version: 1.2.6
5
5
  platform: java
6
6
  authors:
7
7
  - Brian Stien
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: exe
14
14
  cert_chain: []
15
- date: 2020-05-19 00:00:00.000000000 Z
15
+ date: 2021-09-29 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  requirement: !ruby/object:Gem::Requirement
@@ -21,8 +21,8 @@ dependencies:
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2.7'
23
23
  name: march_hare
24
- prerelease: false
25
24
  type: :runtime
25
+ prerelease: false
26
26
  version_requirements: !ruby/object:Gem::Requirement
27
27
  requirements:
28
28
  - - "~>"
@@ -35,8 +35,8 @@ dependencies:
35
35
  - !ruby/object:Gem::Version
36
36
  version: '3.2'
37
37
  name: activesupport
38
- prerelease: false
39
38
  type: :runtime
39
+ prerelease: false
40
40
  version_requirements: !ruby/object:Gem::Requirement
41
41
  requirements:
42
42
  - - ">="
@@ -49,8 +49,8 @@ dependencies:
49
49
  - !ruby/object:Gem::Version
50
50
  version: '0'
51
51
  name: concurrent-ruby
52
- prerelease: false
53
52
  type: :runtime
53
+ prerelease: false
54
54
  version_requirements: !ruby/object:Gem::Requirement
55
55
  requirements:
56
56
  - - ">="
@@ -63,8 +63,8 @@ dependencies:
63
63
  - !ruby/object:Gem::Version
64
64
  version: 0.2.0
65
65
  name: multi_op_queue
66
- prerelease: false
67
66
  type: :runtime
67
+ prerelease: false
68
68
  version_requirements: !ruby/object:Gem::Requirement
69
69
  requirements:
70
70
  - - ">="
@@ -77,8 +77,8 @@ dependencies:
77
77
  - !ruby/object:Gem::Version
78
78
  version: '0'
79
79
  name: benchmark-ips
80
- prerelease: false
81
80
  type: :development
81
+ prerelease: false
82
82
  version_requirements: !ruby/object:Gem::Requirement
83
83
  requirements:
84
84
  - - ">="
@@ -91,8 +91,8 @@ dependencies:
91
91
  - !ruby/object:Gem::Version
92
92
  version: '0'
93
93
  name: bundler
94
- prerelease: false
95
94
  type: :development
95
+ prerelease: false
96
96
  version_requirements: !ruby/object:Gem::Requirement
97
97
  requirements:
98
98
  - - ">="
@@ -105,8 +105,8 @@ dependencies:
105
105
  - !ruby/object:Gem::Version
106
106
  version: '0'
107
107
  name: connection_pool
108
- prerelease: false
109
108
  type: :development
109
+ prerelease: false
110
110
  version_requirements: !ruby/object:Gem::Requirement
111
111
  requirements:
112
112
  - - ">="
@@ -119,8 +119,8 @@ dependencies:
119
119
  - !ruby/object:Gem::Version
120
120
  version: '0'
121
121
  name: fakeredis
122
- prerelease: false
123
122
  type: :development
123
+ prerelease: false
124
124
  version_requirements: !ruby/object:Gem::Requirement
125
125
  requirements:
126
126
  - - ">="
@@ -133,8 +133,8 @@ dependencies:
133
133
  - !ruby/object:Gem::Version
134
134
  version: '0'
135
135
  name: pry
136
- prerelease: false
137
136
  type: :development
137
+ prerelease: false
138
138
  version_requirements: !ruby/object:Gem::Requirement
139
139
  requirements:
140
140
  - - ">="
@@ -147,8 +147,8 @@ dependencies:
147
147
  - !ruby/object:Gem::Version
148
148
  version: '0'
149
149
  name: rake
150
- prerelease: false
151
150
  type: :development
151
+ prerelease: false
152
152
  version_requirements: !ruby/object:Gem::Requirement
153
153
  requirements:
154
154
  - - ">="
@@ -161,8 +161,8 @@ dependencies:
161
161
  - !ruby/object:Gem::Version
162
162
  version: '3.2'
163
163
  name: rspec
164
- prerelease: false
165
164
  type: :development
165
+ prerelease: false
166
166
  version_requirements: !ruby/object:Gem::Requirement
167
167
  requirements:
168
168
  - - "~>"
@@ -179,9 +179,9 @@ executables: []
179
179
  extensions: []
180
180
  extra_rdoc_files: []
181
181
  files:
182
+ - ".circleci/config.yml"
182
183
  - ".gitignore"
183
184
  - ".rspec"
184
- - ".travis.yml"
185
185
  - CODE_OF_CONDUCT.md
186
186
  - Gemfile
187
187
  - LICENSE.txt
@@ -223,8 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
223
223
  - !ruby/object:Gem::Version
224
224
  version: '0'
225
225
  requirements: []
226
- rubyforge_project:
227
- rubygems_version: 2.7.9
226
+ rubygems_version: 3.0.6
228
227
  signing_key:
229
228
  specification_version: 4
230
229
  summary: Aims to make publishing work across MRI and jRuby painless and add some nice
data/.travis.yml DELETED
@@ -1,14 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.3.8
4
- - 2.5.7
5
- - jruby-9.1.12.0
6
- - jruby-9.2.7.0
7
- addons:
8
- apt:
9
- packages:
10
- - rabbitmq-server
11
- services:
12
- - rabbitmq
13
- sudo: false
14
- cache: bundler