hanami-events-cloud_pubsub 2.8.1 → 3.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/.rubocop_todo.yml +14 -10
- data/Gemfile +2 -0
- data/Gemfile.lock +50 -42
- data/README.md +13 -0
- data/examples/client.rb +1 -1
- data/examples/server.rb +2 -0
- data/hanami-events-cloud_pubsub.gemspec +4 -1
- data/lib/hanami/events/adapter/cloud_pubsub.rb +29 -11
- data/lib/hanami/events/cloud_pubsub.rb +25 -4
- data/lib/hanami/events/cloud_pubsub/cli.rb +2 -2
- data/lib/hanami/events/cloud_pubsub/listener.rb +35 -10
- data/lib/hanami/events/cloud_pubsub/middleware/client/request_id.rb +22 -0
- data/lib/hanami/events/cloud_pubsub/middleware/request_id.rb +41 -0
- data/lib/hanami/events/cloud_pubsub/middleware/stack.rb +3 -3
- data/lib/hanami/events/cloud_pubsub/runner.rb +4 -8
- data/lib/hanami/events/cloud_pubsub/thread_inspector.rb +1 -1
- data/lib/hanami/events/cloud_pubsub/version.rb +1 -1
- metadata +25 -10
- data/lib/hanami/events/cloud_pubsub/middleware/auto_retry.rb +0 -93
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5112da85db469fae18a876b6f4a98ae0f70c557b215c6ce5cd09294a98cb56be
|
4
|
+
data.tar.gz: 9b1ac9101488c8e42104df3ce9075c641b9d634932c212ee9da7f837fa1449b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7aa2d8aca9fb271a9c7ee5197858f46ca66e65e0b098dfad3015fb982c346f779792f5b6184d25ad64731c34e7aa1394cbc08acbb00db7e441a23d9496b4fca8
|
7
|
+
data.tar.gz: 85564ccda38f3e5ca26e8dc4719961b1b7483d3d5a56e7387af9bf39f321159dfd4bbf0f488570aa37fdfee3ebb16610d49752660828ee7618cddbf76d9f8185
|
data/.rubocop.yml
CHANGED
data/.rubocop_todo.yml
CHANGED
@@ -1,13 +1,24 @@
|
|
1
1
|
# This configuration was generated by
|
2
2
|
# `rubocop --auto-gen-config`
|
3
|
-
# on
|
3
|
+
# on 2020-09-25 19:54:44 UTC using RuboCop version 0.89.0.
|
4
4
|
# The point is for the user to remove these configuration records
|
5
5
|
# one by one as the offenses are removed from the code base.
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
8
8
|
|
9
|
-
# Offense count:
|
10
|
-
#
|
9
|
+
# Offense count: 2
|
10
|
+
# Cop supports --auto-correct.
|
11
|
+
Lint/RedundantCopDisableDirective:
|
12
|
+
Exclude:
|
13
|
+
- 'lib/hanami/events/adapter/cloud_pubsub.rb'
|
14
|
+
|
15
|
+
# Offense count: 3
|
16
|
+
# Configuration parameters: IgnoredMethods.
|
17
|
+
Metrics/AbcSize:
|
18
|
+
Max: 21
|
19
|
+
|
20
|
+
# Offense count: 4
|
21
|
+
# Configuration parameters: CountComments, CountAsOne, ExcludedMethods.
|
11
22
|
Metrics/MethodLength:
|
12
23
|
Max: 14
|
13
24
|
|
@@ -16,10 +27,3 @@ Metrics/MethodLength:
|
|
16
27
|
Style/GlobalVars:
|
17
28
|
Exclude:
|
18
29
|
- 'examples/server.rb'
|
19
|
-
|
20
|
-
# Offense count: 35
|
21
|
-
# Cop supports --auto-correct.
|
22
|
-
# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
23
|
-
# URISchemes: http, https
|
24
|
-
Metrics/LineLength:
|
25
|
-
Max: 96
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -8,9 +8,10 @@ GIT
|
|
8
8
|
PATH
|
9
9
|
remote: .
|
10
10
|
specs:
|
11
|
-
hanami-events-cloud_pubsub (
|
11
|
+
hanami-events-cloud_pubsub (3.0.3)
|
12
12
|
dry-configurable (>= 0.8)
|
13
|
-
|
13
|
+
gapic-common (>= 0.3.4)
|
14
|
+
google-cloud-pubsub (>= 0.38.1, < 2.4)
|
14
15
|
hanami-cli (~> 0.2)
|
15
16
|
hanami-events (~> 0.2.0)
|
16
17
|
rack
|
@@ -20,12 +21,12 @@ GEM
|
|
20
21
|
specs:
|
21
22
|
addressable (2.7.0)
|
22
23
|
public_suffix (>= 2.0.2, < 5.0)
|
23
|
-
ast (2.4.
|
24
|
-
coderay (1.1.
|
24
|
+
ast (2.4.1)
|
25
|
+
coderay (1.1.3)
|
25
26
|
concurrent-ruby (1.1.6)
|
26
|
-
diff-lcs (1.
|
27
|
+
diff-lcs (1.4.4)
|
27
28
|
docile (1.3.2)
|
28
|
-
dry-configurable (0.11.
|
29
|
+
dry-configurable (0.11.6)
|
29
30
|
concurrent-ruby (~> 1.0)
|
30
31
|
dry-core (~> 0.4, >= 0.4.7)
|
31
32
|
dry-equalizer (~> 0.2)
|
@@ -35,43 +36,45 @@ GEM
|
|
35
36
|
dry-core (0.4.9)
|
36
37
|
concurrent-ruby (~> 1.0)
|
37
38
|
dry-equalizer (0.3.0)
|
38
|
-
faraday (1.0
|
39
|
+
faraday (1.1.0)
|
39
40
|
multipart-post (>= 1.2, < 3)
|
41
|
+
ruby2_keywords
|
42
|
+
gapic-common (0.3.4)
|
43
|
+
google-protobuf (~> 3.12, >= 3.12.2)
|
44
|
+
googleapis-common-protos (>= 1.3.9, < 2.0)
|
45
|
+
googleapis-common-protos-types (>= 1.0.4, < 2.0)
|
46
|
+
googleauth (~> 0.9)
|
47
|
+
grpc (~> 1.25)
|
40
48
|
google-cloud-core (1.5.0)
|
41
49
|
google-cloud-env (~> 1.0)
|
42
50
|
google-cloud-errors (~> 1.0)
|
43
|
-
google-cloud-env (1.
|
51
|
+
google-cloud-env (1.4.0)
|
44
52
|
faraday (>= 0.17.3, < 2.0)
|
45
53
|
google-cloud-errors (1.0.1)
|
46
|
-
google-cloud-pubsub (
|
54
|
+
google-cloud-pubsub (2.3.0)
|
47
55
|
concurrent-ruby (~> 1.1)
|
48
|
-
google-cloud-core (~> 1.
|
49
|
-
google-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
googleapis-common-protos (>= 1.3.9, < 2.0)
|
56
|
-
googleauth (~> 0.9)
|
57
|
-
grpc (~> 1.24)
|
58
|
-
rly (~> 0.2.3)
|
59
|
-
google-protobuf (3.12.2)
|
56
|
+
google-cloud-core (~> 1.5)
|
57
|
+
google-cloud-pubsub-v1 (~> 0.0)
|
58
|
+
google-cloud-pubsub-v1 (0.1.2)
|
59
|
+
gapic-common (~> 0.3)
|
60
|
+
google-cloud-errors (~> 1.0)
|
61
|
+
grpc-google-iam-v1 (>= 0.6.10, < 2.0)
|
62
|
+
google-protobuf (3.14.0)
|
60
63
|
googleapis-common-protos (1.3.10)
|
61
64
|
google-protobuf (~> 3.11)
|
62
65
|
googleapis-common-protos-types (>= 1.0.5, < 2.0)
|
63
66
|
grpc (~> 1.27)
|
64
67
|
googleapis-common-protos-types (1.0.5)
|
65
68
|
google-protobuf (~> 3.11)
|
66
|
-
googleauth (0.
|
69
|
+
googleauth (0.14.0)
|
67
70
|
faraday (>= 0.17.3, < 2.0)
|
68
71
|
jwt (>= 1.4, < 3.0)
|
69
72
|
memoist (~> 0.16)
|
70
73
|
multi_json (~> 1.11)
|
71
74
|
os (>= 0.9, < 2.0)
|
72
75
|
signet (~> 0.14)
|
73
|
-
grpc (1.
|
74
|
-
google-protobuf (~> 3.
|
76
|
+
grpc (1.34.0)
|
77
|
+
google-protobuf (~> 3.13)
|
75
78
|
googleapis-common-protos-types (~> 1.0)
|
76
79
|
grpc-google-iam-v1 (0.6.10)
|
77
80
|
google-protobuf (~> 3.11)
|
@@ -83,48 +86,52 @@ GEM
|
|
83
86
|
hanami-utils (1.3.6)
|
84
87
|
concurrent-ruby (~> 1.0)
|
85
88
|
transproc (~> 1.0)
|
86
|
-
|
87
|
-
jwt (2.2.1)
|
89
|
+
jwt (2.2.2)
|
88
90
|
memoist (0.16.2)
|
89
91
|
method_source (1.0.0)
|
90
|
-
multi_json (1.
|
92
|
+
multi_json (1.15.0)
|
91
93
|
multipart-post (2.1.1)
|
92
|
-
os (1.1.
|
93
|
-
parallel (1.19.
|
94
|
-
parser (2.7.1.
|
95
|
-
ast (~> 2.4.
|
96
|
-
prometheus-client (2.
|
94
|
+
os (1.1.1)
|
95
|
+
parallel (1.19.2)
|
96
|
+
parser (2.7.1.4)
|
97
|
+
ast (~> 2.4.1)
|
98
|
+
prometheus-client (2.1.0)
|
97
99
|
pry (0.13.1)
|
98
100
|
coderay (~> 1.1)
|
99
101
|
method_source (~> 1.0)
|
100
|
-
public_suffix (4.0.
|
101
|
-
rack (2.2.
|
102
|
+
public_suffix (4.0.6)
|
103
|
+
rack (2.2.3)
|
102
104
|
rainbow (3.0.0)
|
103
105
|
rake (13.0.1)
|
106
|
+
regexp_parser (1.7.1)
|
107
|
+
request_id (0.4.3)
|
104
108
|
rexml (3.2.4)
|
105
|
-
rly (0.2.3)
|
106
109
|
rspec (3.9.0)
|
107
110
|
rspec-core (~> 3.9.0)
|
108
111
|
rspec-expectations (~> 3.9.0)
|
109
112
|
rspec-mocks (~> 3.9.0)
|
110
|
-
rspec-core (3.9.
|
111
|
-
rspec-support (~> 3.9.
|
112
|
-
rspec-expectations (3.9.
|
113
|
+
rspec-core (3.9.2)
|
114
|
+
rspec-support (~> 3.9.3)
|
115
|
+
rspec-expectations (3.9.2)
|
113
116
|
diff-lcs (>= 1.2.0, < 2.0)
|
114
117
|
rspec-support (~> 3.9.0)
|
115
118
|
rspec-mocks (3.9.1)
|
116
119
|
diff-lcs (>= 1.2.0, < 2.0)
|
117
120
|
rspec-support (~> 3.9.0)
|
118
|
-
rspec-support (3.9.
|
119
|
-
rubocop (0.
|
120
|
-
jaro_winkler (~> 1.5.1)
|
121
|
+
rspec-support (3.9.3)
|
122
|
+
rubocop (0.89.0)
|
121
123
|
parallel (~> 1.10)
|
122
|
-
parser (>= 2.7.
|
124
|
+
parser (>= 2.7.1.1)
|
123
125
|
rainbow (>= 2.2.2, < 4.0)
|
126
|
+
regexp_parser (>= 1.7)
|
124
127
|
rexml
|
128
|
+
rubocop-ast (>= 0.1.0, < 1.0)
|
125
129
|
ruby-progressbar (~> 1.7)
|
126
130
|
unicode-display_width (>= 1.4.0, < 2.0)
|
131
|
+
rubocop-ast (0.3.0)
|
132
|
+
parser (>= 2.7.1.4)
|
127
133
|
ruby-progressbar (1.10.1)
|
134
|
+
ruby2_keywords (0.0.2)
|
128
135
|
signet (0.14.0)
|
129
136
|
addressable (~> 2.3)
|
130
137
|
faraday (>= 0.17.3, < 2.0)
|
@@ -147,6 +154,7 @@ DEPENDENCIES
|
|
147
154
|
prometheus-client
|
148
155
|
pry
|
149
156
|
rake (~> 13.0)
|
157
|
+
request_id (~> 0.4.3)
|
150
158
|
rspec (~> 3.0)
|
151
159
|
rubocop
|
152
160
|
simplecov
|
data/README.md
CHANGED
@@ -106,6 +106,19 @@ Then, run the worker process:
|
|
106
106
|
bundle exec cloudpubsub run
|
107
107
|
```
|
108
108
|
|
109
|
+
# `RequestId` Integration
|
110
|
+
|
111
|
+
To gem has an integration with [request_id](https://github.com/remind101/request_id), so you can trace interactions between various components in your app. To make it work properly, make sure the `request_id` is added to your Gemfile, then require the middleware:
|
112
|
+
|
113
|
+
```ruby
|
114
|
+
require 'hanami/events/cloud_pubsub/middleware/request_id'
|
115
|
+
|
116
|
+
Hanami::Events::CloudPubsub.configure do |config|
|
117
|
+
# ...
|
118
|
+
end
|
119
|
+
```
|
120
|
+
|
121
|
+
|
109
122
|
# Prometheus Integration
|
110
123
|
|
111
124
|
If you have the `prometheus-client` gem installed, a `/metrics` endpoint will
|
data/examples/client.rb
CHANGED
@@ -12,7 +12,7 @@ Hanami::Events::CloudPubsub.setup
|
|
12
12
|
|
13
13
|
pubsub = Google::Cloud::Pubsub.new project_id: 'emulator'
|
14
14
|
|
15
|
-
events = Hanami::Events.initialize(:cloud_pubsub, pubsub: pubsub, logger: Logger.new(
|
15
|
+
events = Hanami::Events.initialize(:cloud_pubsub, pubsub: pubsub, logger: Logger.new($stdout))
|
16
16
|
|
17
17
|
3.times do
|
18
18
|
events.broadcast('user.deleted', user_id: 1)
|
data/examples/server.rb
CHANGED
@@ -4,6 +4,8 @@ Hanami::Events::CloudPubsub.configure do |config|
|
|
4
4
|
config.subscriber.streams = 2
|
5
5
|
config.subscriber.threads.push = 2
|
6
6
|
config.subscriber.threads.callback = 2
|
7
|
+
config.auto_retry.enabled = true
|
8
|
+
config.auto_retry.dead_letter_topic_name = 'my-dead-letter-topic-name'
|
7
9
|
|
8
10
|
config.subscriptions_loader = -> do
|
9
11
|
$events.subscribe('user.deleted', id: 'testing-2') do |payload|
|
@@ -15,6 +15,8 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.homepage = 'https://github.com/adHawk/hanami-events-cloud_pubsub'
|
16
16
|
spec.license = 'MIT'
|
17
17
|
|
18
|
+
spec.required_ruby_version = '>= 2.5.0'
|
19
|
+
|
18
20
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
19
21
|
f.match(%r{^(test|spec|features)/})
|
20
22
|
end
|
@@ -23,7 +25,8 @@ Gem::Specification.new do |spec|
|
|
23
25
|
spec.require_paths = ['lib']
|
24
26
|
|
25
27
|
spec.add_dependency 'dry-configurable', '>= 0.8'
|
26
|
-
spec.add_dependency '
|
28
|
+
spec.add_dependency 'gapic-common', '>= 0.3.4'
|
29
|
+
spec.add_dependency 'google-cloud-pubsub', '>= 0.38.1', '< 2.4'
|
27
30
|
spec.add_dependency 'hanami-cli', '~> 0.2'
|
28
31
|
spec.add_dependency 'hanami-events', '~> 0.2.0'
|
29
32
|
spec.add_dependency 'rack'
|
@@ -11,31 +11,40 @@ module Hanami
|
|
11
11
|
#
|
12
12
|
# @api private
|
13
13
|
class CloudPubsub
|
14
|
-
attr_reader :subscribers, :listeners, :topic_registry
|
14
|
+
attr_reader :subscribers, :listeners, :topic_registry, :middleware
|
15
15
|
|
16
16
|
def initialize(params)
|
17
17
|
@pubsub = params[:pubsub]
|
18
|
-
@logger = params[:logger] || Logger.new(
|
18
|
+
@logger = params[:logger] || Logger.new($stdout)
|
19
19
|
@listen = params[:listen] || false
|
20
20
|
@subscribers = Concurrent::Array.new
|
21
21
|
@listeners = Concurrent::Array.new
|
22
22
|
@serializer_type = params.fetch(:serializer, :json).to_sym
|
23
23
|
@topic_registry = {}
|
24
24
|
@mutex = Mutex.new
|
25
|
+
@middleware = ::Hanami::Events::CloudPubsub.config.client_middleware
|
25
26
|
end
|
26
27
|
|
27
28
|
# Brodcasts event to all subscribes
|
28
29
|
#
|
29
30
|
# @param event [Symbol, String] the event name
|
30
31
|
# @param payload [Hash] the event data
|
31
|
-
def broadcast(name,
|
32
|
+
def broadcast(name, input_payload, **message_opts)
|
32
33
|
event_name = namespaced(name)
|
33
34
|
topic = topic_for event_name
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
35
|
+
serialized_payload = serializer.serialize(input_payload)
|
36
|
+
attrs = { id: SecureRandom.uuid, event_name: event_name }
|
37
|
+
|
38
|
+
middleware.invoke(serialized_payload, **attrs, **message_opts) do |payload, **opts|
|
39
|
+
topic.publish_async(payload, **opts) do |result|
|
40
|
+
msg = result.message.grpc.to_h
|
41
|
+
|
42
|
+
if result.succeeded?
|
43
|
+
logger.info "Published #{name.inspect} published", **msg
|
44
|
+
else
|
45
|
+
logger.warn "Failed to broadcast #{name.inspect} event", error: result.error, **msg # rubocop:disable Layout/LineLength
|
46
|
+
end
|
47
|
+
end
|
39
48
|
end
|
40
49
|
end
|
41
50
|
|
@@ -74,7 +83,8 @@ module Hanami
|
|
74
83
|
handler: method(:call_subscribers),
|
75
84
|
logger: logger,
|
76
85
|
topic: topic,
|
77
|
-
subscriber_opts: subscriber_opts
|
86
|
+
subscriber_opts: subscriber_opts,
|
87
|
+
dead_letter_topic: dead_letter_topic
|
78
88
|
)
|
79
89
|
|
80
90
|
@listeners << listener
|
@@ -95,7 +105,7 @@ module Hanami
|
|
95
105
|
@serializer ||= Hanami::Events::Serializer[@serializer_type].new
|
96
106
|
end
|
97
107
|
|
98
|
-
# rubocop:disable
|
108
|
+
# rubocop:disable Layout/LineLength
|
99
109
|
def topic_for(name)
|
100
110
|
return @topic_registry[name.to_s] if @topic_registry[name.to_s]
|
101
111
|
|
@@ -107,7 +117,15 @@ module Hanami
|
|
107
117
|
|
108
118
|
@topic_registry[name.to_s] = topic
|
109
119
|
end
|
110
|
-
# rubocop:enable
|
120
|
+
# rubocop:enable Layout/LineLength
|
121
|
+
|
122
|
+
def dead_letter_topic
|
123
|
+
conf = Hanami::Events::CloudPubsub.config.auto_retry
|
124
|
+
|
125
|
+
return unless conf.enabled
|
126
|
+
|
127
|
+
topic_for namespaced(conf.dead_letter_topic_name)
|
128
|
+
end
|
111
129
|
|
112
130
|
def namespaced(val, sep: '.')
|
113
131
|
[Hanami::Events::CloudPubsub.namespace, val].compact.join(sep)
|
@@ -5,7 +5,6 @@ require 'hanami/events'
|
|
5
5
|
require 'hanami/events/cloud_pubsub/version'
|
6
6
|
require 'hanami/events/cloud_pubsub/middleware/stack'
|
7
7
|
require 'hanami/events/cloud_pubsub/middleware/logging'
|
8
|
-
require 'hanami/events/cloud_pubsub/middleware/auto_retry'
|
9
8
|
require 'hanami/events/cloud_pubsub/runner'
|
10
9
|
require 'hanami/events/cloud_pubsub/errors'
|
11
10
|
require 'google/cloud/pubsub'
|
@@ -32,7 +31,7 @@ module Hanami
|
|
32
31
|
setting :project_id, reader: true
|
33
32
|
setting :auto_create_subscriptions, false, reader: true
|
34
33
|
setting :auto_create_topics, false, reader: true
|
35
|
-
setting :logger, Logger.new(
|
34
|
+
setting :logger, Logger.new($stdout), reader: true
|
36
35
|
setting :subscriptions_loader, proc {
|
37
36
|
abort <<~MSG
|
38
37
|
┌────────────────────────────────────────────────────────────────────────────────┐
|
@@ -58,8 +57,7 @@ module Hanami
|
|
58
57
|
], reader: true
|
59
58
|
|
60
59
|
middleware_stack = Middleware::Stack.new(
|
61
|
-
Middleware::Logging.new
|
62
|
-
Middleware::AutoRetry.new
|
60
|
+
Middleware::Logging.new
|
63
61
|
)
|
64
62
|
|
65
63
|
begin
|
@@ -72,8 +70,31 @@ module Hanami
|
|
72
70
|
|
73
71
|
setting :middleware, middleware_stack
|
74
72
|
|
73
|
+
client_middleware_stack = Middleware::Stack.new
|
74
|
+
|
75
|
+
begin
|
76
|
+
require 'request_id'
|
77
|
+
require 'hanami/events/cloud_pubsub/middleware/client/request_id'
|
78
|
+
require 'hanami/events/cloud_pubsub/middleware/request_id'
|
79
|
+
|
80
|
+
client_middleware_stack.prepend(Middleware::Client::RequestId.new)
|
81
|
+
middleware_stack.prepend(Middleware::RequestId.new)
|
82
|
+
rescue LoadError
|
83
|
+
# ok
|
84
|
+
end
|
85
|
+
|
86
|
+
setting :client_middleware, client_middleware_stack
|
87
|
+
|
75
88
|
setting :on_shutdown_handlers, [], reader: true
|
76
89
|
|
90
|
+
setting :auto_retry do
|
91
|
+
setting :enabled, false
|
92
|
+
setting :max_attempts, 1200
|
93
|
+
setting :dead_letter_topic_name
|
94
|
+
setting :minimum_backoff, 30
|
95
|
+
setting :maximum_backoff, 600
|
96
|
+
end
|
97
|
+
|
77
98
|
def self.finalize_settings!
|
78
99
|
conf_hash = config.pubsub
|
79
100
|
conf_hash.each { |key, val| Google::Cloud::Pubsub.configure[key] = val }
|
@@ -5,6 +5,7 @@ require 'hanami/events/cloud_pubsub/safe_error_handler'
|
|
5
5
|
module Hanami
|
6
6
|
module Events
|
7
7
|
module CloudPubsub
|
8
|
+
# rubocop:disable Metrics/ClassLength:
|
8
9
|
# @api private
|
9
10
|
class Listener
|
10
11
|
attr_reader :topic,
|
@@ -13,8 +14,10 @@ module Hanami
|
|
13
14
|
:logger,
|
14
15
|
:handler,
|
15
16
|
:event_name,
|
16
|
-
:
|
17
|
-
:middleware
|
17
|
+
:input_subscriber_opts,
|
18
|
+
:middleware,
|
19
|
+
:dead_letter_topic
|
20
|
+
|
18
21
|
# rubocop:disable Metrics/ParameterLists
|
19
22
|
def initialize(topic:,
|
20
23
|
logger:,
|
@@ -22,25 +25,24 @@ module Hanami
|
|
22
25
|
event_name:,
|
23
26
|
subscriber_id:,
|
24
27
|
subscriber_opts: {},
|
25
|
-
middleware: CloudPubsub.config.middleware
|
28
|
+
middleware: CloudPubsub.config.middleware,
|
29
|
+
dead_letter_topic: nil)
|
26
30
|
@topic = topic
|
27
31
|
@logger = logger
|
28
32
|
@handler = handler
|
29
33
|
@event_name = event_name
|
30
34
|
@subscriber_id = subscriber_id
|
31
|
-
@
|
35
|
+
@input_subscriber_opts = subscriber_opts
|
32
36
|
@middleware = middleware
|
37
|
+
@dead_letter_topic = dead_letter_topic
|
33
38
|
end
|
34
39
|
# rubocop:enable Metrics/ParameterLists
|
35
40
|
|
36
41
|
def register
|
37
42
|
subscription = subscription_for(subscriber_id)
|
38
|
-
|
39
|
-
listener = subscription.listen(
|
40
|
-
|
41
|
-
end
|
42
|
-
|
43
|
-
logger.debug("Registered listener for #{subscriber_id} with opts #{subscriber_opts}")
|
43
|
+
apply_retry_options(subscription)
|
44
|
+
listener = subscription.listen(**subscriber_options) { |m| handle_message(m) }
|
45
|
+
logger.debug("Registered listener for #{subscriber_id} with: #{subscriber_options}")
|
44
46
|
|
45
47
|
@subscriber = listener
|
46
48
|
|
@@ -83,8 +85,10 @@ module Hanami
|
|
83
85
|
|
84
86
|
def run_handler(message)
|
85
87
|
middleware.invoke(message) { handler.call(message) }
|
88
|
+
message.ack!
|
86
89
|
rescue StandardError => e
|
87
90
|
run_error_handlers(e, message)
|
91
|
+
message.nack! if CloudPubsub.config.auto_retry.enabled
|
88
92
|
raise
|
89
93
|
end
|
90
94
|
|
@@ -120,7 +124,28 @@ module Hanami
|
|
120
124
|
"a subscription already exists for #{sub_name} " \
|
121
125
|
"but its name #{found_subscription.topic.name} does not match #{@event_name}"
|
122
126
|
end
|
127
|
+
|
128
|
+
def subscriber_options
|
129
|
+
@subscriber_options ||= {
|
130
|
+
**CloudPubsub.config.subscriber.to_h,
|
131
|
+
**input_subscriber_opts
|
132
|
+
}
|
133
|
+
end
|
134
|
+
|
135
|
+
def apply_retry_options(sub)
|
136
|
+
return {} unless CloudPubsub.config.auto_retry.enabled
|
137
|
+
|
138
|
+
sub.retry_policy = Google::Cloud::PubSub::RetryPolicy.new(
|
139
|
+
minimum_backoff: CloudPubsub.config.auto_retry.minimum_backoff,
|
140
|
+
maximum_backoff: CloudPubsub.config.auto_retry.maximum_backoff
|
141
|
+
)
|
142
|
+
sub.dead_letter_topic = dead_letter_topic
|
143
|
+
sub.dead_letter_max_delivery_attempts = CloudPubsub.config.auto_retry.max_attempts
|
144
|
+
|
145
|
+
sub
|
146
|
+
end
|
123
147
|
end
|
148
|
+
# rubocop:enable Metrics/ClassLength:
|
124
149
|
end
|
125
150
|
end
|
126
151
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Hanami
|
4
|
+
module Events
|
5
|
+
module CloudPubsub
|
6
|
+
module Middleware
|
7
|
+
module Client
|
8
|
+
# Broadcasts events with current request id
|
9
|
+
class RequestId
|
10
|
+
def call(payload, attributes = {})
|
11
|
+
attributes.merge!(
|
12
|
+
request_id: ::RequestId.request_id || SecureRandom.uuid
|
13
|
+
)
|
14
|
+
|
15
|
+
yield(payload, **attributes)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'request_id'
|
5
|
+
rescue LoadError
|
6
|
+
abort 'Please add "request_id" to your Gemfile to use this middleware'
|
7
|
+
end
|
8
|
+
|
9
|
+
module Hanami
|
10
|
+
module Events
|
11
|
+
module CloudPubsub
|
12
|
+
module Middleware
|
13
|
+
# Middleware used for logging useful information about an event
|
14
|
+
class RequestId
|
15
|
+
def call(msg, **opts)
|
16
|
+
setup_request_id(msg)
|
17
|
+
yield(**opts)
|
18
|
+
ensure
|
19
|
+
clear_request_id(msg)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def setup_request_id(msg)
|
25
|
+
id = msg.attributes['request_id'] || msg.attributes[:request_id]
|
26
|
+
|
27
|
+
::RequestId.request_id = id || SecureRandom.uuid
|
28
|
+
rescue StandardError => e
|
29
|
+
warn "Could not set request_id (#{e.message})"
|
30
|
+
end
|
31
|
+
|
32
|
+
def clear_request_id(_msg)
|
33
|
+
::RequestId.request_id = nil
|
34
|
+
rescue StandardError => e
|
35
|
+
warn "Could not clear request_id (#{e.message})"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -40,14 +40,14 @@ module Hanami
|
|
40
40
|
@entries.shift
|
41
41
|
end
|
42
42
|
|
43
|
-
def invoke(
|
43
|
+
def invoke(*args, **kwargs)
|
44
44
|
stack = entries.dup
|
45
45
|
|
46
46
|
traverse_stack = proc do |**opts|
|
47
47
|
if stack.empty?
|
48
|
-
yield(
|
48
|
+
yield(*args, **kwargs, **opts)
|
49
49
|
else
|
50
|
-
stack.shift.call(
|
50
|
+
stack.shift.call(*args, **kwargs, **opts, &traverse_stack)
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
@@ -86,13 +86,12 @@ module Hanami
|
|
86
86
|
# some reason.
|
87
87
|
#
|
88
88
|
# See: https://github.com/mperham/sidekiq/blob/e447dae961ebc894f12848d9f33446a07ffc67dc/bin/sidekiqload#L74
|
89
|
-
# rubocop:disable Metrics/AbcSize
|
90
89
|
def debug_info
|
91
90
|
<<~MSG
|
92
91
|
╔══════ BACKTRACES
|
93
92
|
#{Thread.list.flat_map { |thr| ThreadInspector.new(thr).to_s }.join("\n")}
|
94
93
|
╠══════ LISTENERS
|
95
|
-
#{adapter.listeners.map { |lis|
|
94
|
+
#{adapter.listeners.map { |lis| "║ #{lis.format}" }.join("\n")}
|
96
95
|
║
|
97
96
|
╠══════ GENERAL
|
98
97
|
║ ready?: #{ready?}
|
@@ -102,7 +101,6 @@ module Hanami
|
|
102
101
|
╚══════
|
103
102
|
MSG
|
104
103
|
end
|
105
|
-
# rubocop:enable Metrics/AbcSize
|
106
104
|
|
107
105
|
def sleep_for_a_bit
|
108
106
|
sleep @sleep_time
|
@@ -114,11 +112,9 @@ module Hanami
|
|
114
112
|
logger.info('Calling custom on_shutdown handler')
|
115
113
|
|
116
114
|
CloudPubsub.on_shutdown_handlers.each do |handler|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
logger.warn("Shutdown handler failed (#{e.message})")
|
121
|
-
end
|
115
|
+
handler.call(adapter)
|
116
|
+
rescue StandardError => e
|
117
|
+
logger.warn("Shutdown handler failed (#{e.message})")
|
122
118
|
end
|
123
119
|
end
|
124
120
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hanami-events-cloud_pubsub
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ian Ker-Seymer
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-06
|
11
|
+
date: 2020-12-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dry-configurable
|
@@ -24,6 +24,20 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0.8'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: gapic-common
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.3.4
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.3.4
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: google-cloud-pubsub
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -33,7 +47,7 @@ dependencies:
|
|
33
47
|
version: 0.38.1
|
34
48
|
- - "<"
|
35
49
|
- !ruby/object:Gem::Version
|
36
|
-
version: '
|
50
|
+
version: '2.4'
|
37
51
|
type: :runtime
|
38
52
|
prerelease: false
|
39
53
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -43,7 +57,7 @@ dependencies:
|
|
43
57
|
version: 0.38.1
|
44
58
|
- - "<"
|
45
59
|
- !ruby/object:Gem::Version
|
46
|
-
version: '
|
60
|
+
version: '2.4'
|
47
61
|
- !ruby/object:Gem::Dependency
|
48
62
|
name: hanami-cli
|
49
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,9 +192,10 @@ files:
|
|
178
192
|
- lib/hanami/events/cloud_pubsub/health_check_server.rb
|
179
193
|
- lib/hanami/events/cloud_pubsub/integration.rb
|
180
194
|
- lib/hanami/events/cloud_pubsub/listener.rb
|
181
|
-
- lib/hanami/events/cloud_pubsub/middleware/
|
195
|
+
- lib/hanami/events/cloud_pubsub/middleware/client/request_id.rb
|
182
196
|
- lib/hanami/events/cloud_pubsub/middleware/logging.rb
|
183
197
|
- lib/hanami/events/cloud_pubsub/middleware/prometheus.rb
|
198
|
+
- lib/hanami/events/cloud_pubsub/middleware/request_id.rb
|
184
199
|
- lib/hanami/events/cloud_pubsub/middleware/stack.rb
|
185
200
|
- lib/hanami/events/cloud_pubsub/register.rb
|
186
201
|
- lib/hanami/events/cloud_pubsub/runner.rb
|
@@ -193,7 +208,7 @@ homepage: https://github.com/adHawk/hanami-events-cloud_pubsub
|
|
193
208
|
licenses:
|
194
209
|
- MIT
|
195
210
|
metadata: {}
|
196
|
-
post_install_message:
|
211
|
+
post_install_message:
|
197
212
|
rdoc_options: []
|
198
213
|
require_paths:
|
199
214
|
- lib
|
@@ -201,15 +216,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
201
216
|
requirements:
|
202
217
|
- - ">="
|
203
218
|
- !ruby/object:Gem::Version
|
204
|
-
version:
|
219
|
+
version: 2.5.0
|
205
220
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
206
221
|
requirements:
|
207
222
|
- - ">="
|
208
223
|
- !ruby/object:Gem::Version
|
209
224
|
version: '0'
|
210
225
|
requirements: []
|
211
|
-
rubygems_version: 3.
|
212
|
-
signing_key:
|
226
|
+
rubygems_version: 3.0.3
|
227
|
+
signing_key:
|
213
228
|
specification_version: 4
|
214
229
|
summary: Google Cloud Pub/Sub adapter for the hanami-events gem
|
215
230
|
test_files: []
|
@@ -1,93 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Hanami
|
4
|
-
module Events
|
5
|
-
module CloudPubsub
|
6
|
-
# Middleware
|
7
|
-
module Middleware
|
8
|
-
# Middleware used for automatically acknowledging messages
|
9
|
-
class AutoRetry
|
10
|
-
attr_reader :max_attempts
|
11
|
-
|
12
|
-
def initialize(logger: nil, max_attempts: 1200)
|
13
|
-
@logger = logger
|
14
|
-
@max_attempts = max_attempts
|
15
|
-
end
|
16
|
-
|
17
|
-
def call(message, **args)
|
18
|
-
succeeded = false
|
19
|
-
failed = false
|
20
|
-
yield(**args)
|
21
|
-
succeeded = true
|
22
|
-
rescue StandardError => e
|
23
|
-
failed = true
|
24
|
-
raise e
|
25
|
-
ensure
|
26
|
-
ack_or_reject(message, succeeded, failed, args)
|
27
|
-
end
|
28
|
-
|
29
|
-
private
|
30
|
-
|
31
|
-
def ack_or_reject(message, succeeded, failed, args)
|
32
|
-
if succeeded
|
33
|
-
handle_success(message, args)
|
34
|
-
elsif failed && max_attempts_reached?(args)
|
35
|
-
handle_max_attempts_reached(message, args)
|
36
|
-
elsif failed
|
37
|
-
handle_failure(message, args)
|
38
|
-
else
|
39
|
-
handle_unfinished(message, args)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def handle_success(message, _args)
|
44
|
-
message.acknowledge!
|
45
|
-
logger.debug "Message(#{message.message_id}) was acknowledged"
|
46
|
-
end
|
47
|
-
|
48
|
-
def max_attempts_reached?(args)
|
49
|
-
args.key?(:attempts) && args[:attempts] >= max_attempts
|
50
|
-
end
|
51
|
-
|
52
|
-
def handle_max_attempts_reached(message, _args)
|
53
|
-
id = message.message_id
|
54
|
-
msg = 'number of attempts exceeded max attempts ' \
|
55
|
-
"of #{max_attempts}, acknowledging message"
|
56
|
-
logger.debug "Message(#{id}) failed, #{msg}"
|
57
|
-
message.acknowledge!
|
58
|
-
end
|
59
|
-
|
60
|
-
def handle_failure(message, args)
|
61
|
-
id = message.message_id
|
62
|
-
seconds = calculate_backoff_seconds(message, args)
|
63
|
-
success = message.modify_ack_deadline!(seconds)
|
64
|
-
msg = "added #{success ? seconds : 0} seconds of delay to ack deadline"
|
65
|
-
logger.debug "Message(#{id}) failed, #{msg}" if success
|
66
|
-
end
|
67
|
-
|
68
|
-
def handle_unfinished(message, _args)
|
69
|
-
id = message.message_id
|
70
|
-
message.reject!
|
71
|
-
logger.warn "Message(#{id}) was terminated from outside, rescheduling"
|
72
|
-
end
|
73
|
-
|
74
|
-
def logger
|
75
|
-
@logger || CloudPubsub.logger
|
76
|
-
end
|
77
|
-
|
78
|
-
def calculate_backoff_seconds(_message, args)
|
79
|
-
amt = if args.key?(:attempts)
|
80
|
-
count = args[:attempts]
|
81
|
-
# min + exponential + random smear
|
82
|
-
15 + count**4 + (rand(30) * (count + 1))
|
83
|
-
else
|
84
|
-
60
|
85
|
-
end
|
86
|
-
|
87
|
-
amt > 600 ? 600 : amt
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|