pwwka 0.22.7.RC → 0.24.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -1
- data/README.md +19 -8
- data/lib/pwwka/configuration.rb +2 -0
- data/lib/pwwka/logging.rb +4 -1
- data/lib/pwwka/transmitter.rb +20 -7
- data/lib/pwwka/version.rb +1 -1
- data/spec/unit/logging_spec.rb +38 -0
- data/spec/unit/transmitter_spec.rb +37 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 91b93a22ea33ffd0151667910c29af13adaf2872c28327ae5af4aff03e9beb64
|
4
|
+
data.tar.gz: 7c8bc1db3fcb3045caafbba21d8ca94838756c347951418535543a7045162ade
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ac5525cd3bdf5a162d0979b58b5f750a3bdd59074d703997bfb5b574f864491430148dd9ba84afe95cd13607ed14244e792f22582004122020411cb6498fb047
|
7
|
+
data.tar.gz: ccdd3938884c1f0d247176baa8c919257154e9126d347e0541703d5b2f860b39d425d8ddc96dfe2fd2fd3a18787a425942bfe7f97daa6e5d48e037188f9507fd
|
data/.circleci/config.yml
CHANGED
@@ -27,6 +27,11 @@ jobs:
|
|
27
27
|
working_directory: "~/pwwka"
|
28
28
|
steps:
|
29
29
|
- checkout
|
30
|
+
- run:
|
31
|
+
name: Check for Gemfile.lock presence
|
32
|
+
command: ' if (test -f Gemfile.lock) then echo "Dont commit Gemfile.lock (see
|
33
|
+
https://github.com/stitchfix/eng-wiki/blob/master/architecture-decisions/0009-rubygem-dependencies-will-be-managed-more-explicitly.md)"
|
34
|
+
1>&2 ; exit 1 ; else exit 0 ; fi '
|
30
35
|
- run: bundle config stitchfix01.jfrog.io $ARTIFACTORY_USER:$ARTIFACTORY_TOKEN
|
31
36
|
- run: bundle install --full-index
|
32
37
|
- run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
|
@@ -51,6 +56,11 @@ jobs:
|
|
51
56
|
working_directory: "~/pwwka"
|
52
57
|
steps:
|
53
58
|
- checkout
|
59
|
+
- run:
|
60
|
+
name: Check for Gemfile.lock presence
|
61
|
+
command: ' if (test -f Gemfile.lock) then echo "Dont commit Gemfile.lock (see
|
62
|
+
https://github.com/stitchfix/eng-wiki/blob/master/architecture-decisions/0009-rubygem-dependencies-will-be-managed-more-explicitly.md)"
|
63
|
+
1>&2 ; exit 1 ; else exit 0 ; fi '
|
54
64
|
- run: bundle config stitchfix01.jfrog.io $ARTIFACTORY_USER:$ARTIFACTORY_TOKEN
|
55
65
|
- run: bundle install --full-index
|
56
66
|
- run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
|
@@ -75,6 +85,11 @@ jobs:
|
|
75
85
|
working_directory: "~/pwwka"
|
76
86
|
steps:
|
77
87
|
- checkout
|
88
|
+
- run:
|
89
|
+
name: Check for Gemfile.lock presence
|
90
|
+
command: ' if (test -f Gemfile.lock) then echo "Dont commit Gemfile.lock (see
|
91
|
+
https://github.com/stitchfix/eng-wiki/blob/master/architecture-decisions/0009-rubygem-dependencies-will-be-managed-more-explicitly.md)"
|
92
|
+
1>&2 ; exit 1 ; else exit 0 ; fi '
|
78
93
|
- run: bundle config stitchfix01.jfrog.io $ARTIFACTORY_USER:$ARTIFACTORY_TOKEN
|
79
94
|
- run: bundle install --full-index
|
80
95
|
- run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
|
@@ -99,6 +114,11 @@ jobs:
|
|
99
114
|
working_directory: "~/pwwka"
|
100
115
|
steps:
|
101
116
|
- checkout
|
117
|
+
- run:
|
118
|
+
name: Check for Gemfile.lock presence
|
119
|
+
command: ' if (test -f Gemfile.lock) then echo "Dont commit Gemfile.lock (see
|
120
|
+
https://github.com/stitchfix/eng-wiki/blob/master/architecture-decisions/0009-rubygem-dependencies-will-be-managed-more-explicitly.md)"
|
121
|
+
1>&2 ; exit 1 ; else exit 0 ; fi '
|
102
122
|
- run: bundle config stitchfix01.jfrog.io $ARTIFACTORY_USER:$ARTIFACTORY_TOKEN
|
103
123
|
- run: bundle install --full-index
|
104
124
|
- run: bundle exec rspec --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
|
@@ -126,7 +146,7 @@ workflows:
|
|
126
146
|
- ruby-2.6.6-rails-5.2
|
127
147
|
filters:
|
128
148
|
tags:
|
129
|
-
only: /^[0-9]+\.[0-9]+\.[0-9]+(\.?(RC|rc)[-\.]?\
|
149
|
+
only: /^[0-9]+\.[0-9]+\.[0-9]+(\.?(RC|rc)[-\.]?\w*)?$/
|
130
150
|
branches:
|
131
151
|
ignore: /.*/
|
132
152
|
- ruby-2.7.1-rails-6.0:
|
data/README.md
CHANGED
@@ -148,8 +148,8 @@ Pwwka::Transmitter.send_message!(payload, routing_key, on_error: :ignore)
|
|
148
148
|
|
149
149
|
#### Delayed Messages
|
150
150
|
|
151
|
-
You might want to delay sending a message (for example, if you have just created a database
|
152
|
-
record and a race condition keeps catching you out). In that case you can use delayed message
|
151
|
+
You might want to delay sending a message (for example, if you have just created a database
|
152
|
+
record and a race condition keeps catching you out). In that case you can use delayed message
|
153
153
|
options:
|
154
154
|
|
155
155
|
```ruby
|
@@ -176,7 +176,12 @@ Pwwka.configure do |config|
|
|
176
176
|
end
|
177
177
|
```
|
178
178
|
|
179
|
-
Regardless of which processor you use, the name of the queue created is `pwwka_send_message_async`.
|
179
|
+
Regardless of which processor you use, the name of the queue created is `pwwka_send_message_async`. You will need to start a worker process to work the queue. For a `Procfile` setup, with Resque as the processor, that could look something like this:
|
180
|
+
|
181
|
+
```ruby
|
182
|
+
pwwka_send_message_async_worker: rake resque:work QUEUE=pwwka_send_message_async
|
183
|
+
```
|
184
|
+
|
180
185
|
|
181
186
|
You can also configure Pwwka to use your own custom job using the `async_job_klass` configuration option. An example might be:
|
182
187
|
```
|
@@ -191,7 +196,7 @@ If you are using Resque and `Resque::Plugins::ExponentialBackoff` is available,
|
|
191
196
|
|
192
197
|
#### Message Queuer
|
193
198
|
|
194
|
-
You can queue up messages and send them in a batch. This is most useful when multiple messages
|
199
|
+
You can queue up messages and send them in a batch. This is most useful when multiple messages
|
195
200
|
need to sent from within a transaction block.
|
196
201
|
|
197
202
|
For example:
|
@@ -255,7 +260,7 @@ Here is an example:
|
|
255
260
|
|
256
261
|
```ruby
|
257
262
|
class ClientIndexMessageHandler
|
258
|
-
|
263
|
+
|
259
264
|
def self.handle!(delivery_info, properties, payload)
|
260
265
|
handler.do_a_thing(payload)
|
261
266
|
end
|
@@ -430,7 +435,7 @@ If you use [Resque][resque], and you wish to handle messages in a resque job, yo
|
|
430
435
|
|
431
436
|
Note that you must provide `@queue` in your job. `QueueResqueJobHandler` doesn't support setting a custom queue at enqueue-time (PRs welcome :).
|
432
437
|
|
433
|
-
Note that if you were using this library before version 0.12.0, your job would only be given the payload. If you change your job to accept
|
438
|
+
Note that if you were using this library before version 0.12.0, your job would only be given the payload. If you change your job to accept exactly three arguments, you will be given the payload, routing key, and message properties. If any of those arguments are optional, you will need to set `PWWKA_QUEUE_EXTENDED_INFO` to `"true"` to force pwwka to pass those along. Without it, your job only gets the payload to avoid breaking legacy consumers.
|
434
439
|
|
435
440
|
3. Profit!
|
436
441
|
|
@@ -454,9 +459,9 @@ describe "my integration test" do
|
|
454
459
|
@test_handler.test_setup
|
455
460
|
end
|
456
461
|
|
457
|
-
after(:all) do
|
462
|
+
after(:all) do
|
458
463
|
# this clears out any messages, so you have a clean test environment next time
|
459
|
-
@test_handler.test_teardown
|
464
|
+
@test_handler.test_teardown
|
460
465
|
end
|
461
466
|
|
462
467
|
it "uses the message bus" do
|
@@ -527,6 +532,12 @@ Pwwka.configuration.payload_logging = :error # Only log payloads for ERROR or FA
|
|
527
532
|
Pwwka.configuration.payload_logging = :fatal # Only log payloads for FATAL messages
|
528
533
|
```
|
529
534
|
|
535
|
+
You can also hook into logging by passing a hash containing keys of strings to match and corresponding `Proc` objects for the logger to execute instead of logging a message. The `Proc` will be called with the original message string that was to be logged and the params specific for that log event. So, if for instance, you wanted to emit a count metric to your monitoring system instead of logging each processed message you could set the configuration:
|
536
|
+
|
537
|
+
```ruby
|
538
|
+
Pwwka.configuration.log_hooks = { 'Processed Message on' => ->(message, params){ $stats.count('message_processed') } }
|
539
|
+
```
|
540
|
+
|
530
541
|
#### Manual monitoring
|
531
542
|
|
532
543
|
RabbitMQ has a web interface for checking out the health of connections, channels, exchanges and queues. Your RabbitMQ provider should
|
data/lib/pwwka/configuration.rb
CHANGED
@@ -10,6 +10,7 @@ module Pwwka
|
|
10
10
|
attr_accessor :delayed_exchange_name
|
11
11
|
attr_accessor :logger
|
12
12
|
attr_accessor :log_level
|
13
|
+
attr_accessor :log_hooks
|
13
14
|
attr_accessor :options
|
14
15
|
attr_accessor :background_job_processor
|
15
16
|
attr_accessor :send_message_resque_backoff_strategy
|
@@ -26,6 +27,7 @@ module Pwwka
|
|
26
27
|
@delayed_exchange_name = "pwwka.delayed.#{Pwwka.environment}"
|
27
28
|
@logger = MonoLogger.new(STDOUT)
|
28
29
|
@log_level = :info
|
30
|
+
@log_hooks = {}
|
29
31
|
@options = {}
|
30
32
|
@send_message_resque_backoff_strategy = [5, #intermittent glitch?
|
31
33
|
60, # quick interruption
|
data/lib/pwwka/logging.rb
CHANGED
@@ -22,7 +22,10 @@ module Pwwka
|
|
22
22
|
params[:payload] = "[omitted]" if params[:payload]
|
23
23
|
end
|
24
24
|
message = format % params
|
25
|
-
|
25
|
+
|
26
|
+
if Pwwka.configuration.log_hooks.select { |key, _value| message.match key }.each { |_key, value| value.call(message, params) }.empty?
|
27
|
+
logger.send(level,message)
|
28
|
+
end
|
26
29
|
end
|
27
30
|
end
|
28
31
|
end
|
data/lib/pwwka/transmitter.rb
CHANGED
@@ -23,10 +23,17 @@ module Pwwka
|
|
23
23
|
|
24
24
|
DEFAULT_DELAY_BY_MS = 5000
|
25
25
|
|
26
|
+
attr_reader :caller_manages_connector
|
26
27
|
attr_reader :channel_connector
|
27
28
|
|
28
|
-
def initialize
|
29
|
-
|
29
|
+
def initialize(channel_connector: nil)
|
30
|
+
if channel_connector
|
31
|
+
@caller_manages_connector = true
|
32
|
+
@channel_connector = channel_connector
|
33
|
+
else
|
34
|
+
@caller_manages_connector = false
|
35
|
+
@channel_connector = ChannelConnector.new(connection_name: "p: #{Pwwka.configuration.app_id} #{Pwwka.configuration.process_name}".strip)
|
36
|
+
end
|
30
37
|
end
|
31
38
|
|
32
39
|
# Send an important message that must go through. This method allows any raised exception
|
@@ -54,11 +61,13 @@ module Pwwka
|
|
54
61
|
delay_by: nil,
|
55
62
|
type: nil,
|
56
63
|
message_id: :auto_generate,
|
57
|
-
headers: nil
|
64
|
+
headers: nil,
|
65
|
+
channel_connector: nil
|
66
|
+
)
|
58
67
|
if delayed
|
59
|
-
new.send_delayed_message!(*[payload, routing_key, delay_by].compact, type: type, headers: headers, message_id: message_id)
|
68
|
+
new(channel_connector: channel_connector).send_delayed_message!(*[payload, routing_key, delay_by].compact, type: type, headers: headers, message_id: message_id)
|
60
69
|
else
|
61
|
-
new.send_message!(payload, routing_key, type: type, headers: headers, message_id: message_id)
|
70
|
+
new(channel_connector: channel_connector).send_message!(payload, routing_key, type: type, headers: headers, message_id: message_id)
|
62
71
|
end
|
63
72
|
logf "AFTER Transmitting Message on %{routing_key} -> %{payload}",routing_key: routing_key, payload: payload
|
64
73
|
true
|
@@ -147,7 +156,9 @@ module Pwwka
|
|
147
156
|
logf "END Transmitting Message on id[%{id}] %{routing_key} -> %{payload}", id: publish_options.message_id, routing_key: routing_key, payload: payload
|
148
157
|
true
|
149
158
|
ensure
|
150
|
-
|
159
|
+
unless caller_manages_connector
|
160
|
+
channel_connector.connection_close
|
161
|
+
end
|
151
162
|
end
|
152
163
|
|
153
164
|
|
@@ -167,7 +178,9 @@ module Pwwka
|
|
167
178
|
logf "END Transmitting Delayed Message on id[%{id}] %{routing_key} -> %{payload}", id: publish_options.message_id, routing_key: routing_key, payload: payload
|
168
179
|
true
|
169
180
|
ensure
|
170
|
-
|
181
|
+
unless caller_manages_connector
|
182
|
+
channel_connector.connection_close
|
183
|
+
end
|
171
184
|
end
|
172
185
|
|
173
186
|
end
|
data/lib/pwwka/version.rb
CHANGED
data/spec/unit/logging_spec.rb
CHANGED
@@ -93,6 +93,44 @@ describe Pwwka::Logging do
|
|
93
93
|
expect(logger).to have_received(:error).with("This is also the payload: [omitted]")
|
94
94
|
end
|
95
95
|
end
|
96
|
+
|
97
|
+
context "log_hook matching" do
|
98
|
+
def test_func(message, params)
|
99
|
+
end
|
100
|
+
|
101
|
+
before do
|
102
|
+
Pwwka.configuration.log_hooks = { "test message" => ->(message, params){ test_func(message, params) } }
|
103
|
+
end
|
104
|
+
|
105
|
+
after do
|
106
|
+
Pwwka.configuration.log_hooks = {}
|
107
|
+
end
|
108
|
+
|
109
|
+
context "message matches hook" do
|
110
|
+
it "overrides logging" do
|
111
|
+
ForLogging.logf("test message", {})
|
112
|
+
|
113
|
+
expect(logger).not_to have_received(:info)
|
114
|
+
end
|
115
|
+
|
116
|
+
it "calls the hook" do
|
117
|
+
allow(self).to receive(:test_func)
|
118
|
+
test_params = { param: :test }
|
119
|
+
|
120
|
+
ForLogging.logf("test message", test_params)
|
121
|
+
|
122
|
+
expect(self).to have_received(:test_func).with("test message", test_params)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context "message doesn't match hook" do
|
127
|
+
it "logs as normal" do
|
128
|
+
ForLogging.logf("other message", {})
|
129
|
+
|
130
|
+
expect(logger).to have_received(:info).with("other message")
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
96
134
|
end
|
97
135
|
|
98
136
|
end
|
@@ -576,5 +576,41 @@ describe Pwwka::Transmitter do
|
|
576
576
|
end
|
577
577
|
end
|
578
578
|
end
|
579
|
-
end
|
580
579
|
|
580
|
+
context "application manages connection" do
|
581
|
+
let(:managed_connector) { instance_double(Pwwka::ChannelConnector,
|
582
|
+
topic_exchange: double(:topic_exchange).as_null_object,
|
583
|
+
delayed_exchange: double(:delayed_exchange).as_null_object
|
584
|
+
).as_null_object }
|
585
|
+
describe ".send_message!" do
|
586
|
+
|
587
|
+
context "send immediate" do
|
588
|
+
it "doesn't open a new connection" do
|
589
|
+
described_class.send_message!(payload, routing_key, delayed: false, channel_connector: managed_connector)
|
590
|
+
|
591
|
+
expect(Pwwka::ChannelConnector).not_to have_received(:new)
|
592
|
+
end
|
593
|
+
|
594
|
+
it "doesn't close a passed in connection" do
|
595
|
+
described_class.send_message!(payload, routing_key, delayed: false, channel_connector: managed_connector)
|
596
|
+
|
597
|
+
expect(managed_connector).not_to have_received(:connection_close)
|
598
|
+
end
|
599
|
+
end
|
600
|
+
|
601
|
+
context "send delayed" do
|
602
|
+
it "doesn't open a new connection" do
|
603
|
+
described_class.send_message!(payload, routing_key, delayed: true, channel_connector: managed_connector)
|
604
|
+
|
605
|
+
expect(Pwwka::ChannelConnector).not_to have_received(:new)
|
606
|
+
end
|
607
|
+
|
608
|
+
it "doesn't close a passed in connection" do
|
609
|
+
described_class.send_message!(payload, routing_key, delayed: true, channel_connector: managed_connector)
|
610
|
+
|
611
|
+
expect(managed_connector).not_to have_received(:connection_close)
|
612
|
+
end
|
613
|
+
end
|
614
|
+
end
|
615
|
+
end
|
616
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pwwka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.24.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stitch Fix Engineering
|
@@ -15,7 +15,7 @@ authors:
|
|
15
15
|
autorequire:
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
|
-
date: 2020-
|
18
|
+
date: 2020-08-25 00:00:00.000000000 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: bunny
|
@@ -315,9 +315,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
315
315
|
version: '0'
|
316
316
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
317
317
|
requirements:
|
318
|
-
- - "
|
318
|
+
- - ">="
|
319
319
|
- !ruby/object:Gem::Version
|
320
|
-
version:
|
320
|
+
version: '0'
|
321
321
|
requirements: []
|
322
322
|
rubygems_version: 3.1.2
|
323
323
|
signing_key:
|