karafka 1.3.0 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.ruby-version +1 -1
- data/.travis.yml +8 -8
- data/CHANGELOG.md +38 -14
- data/Gemfile.lock +55 -52
- data/README.md +3 -3
- data/karafka.gemspec +2 -2
- data/lib/karafka.rb +1 -0
- data/lib/karafka/attributes_map.rb +2 -8
- data/lib/karafka/cli.rb +1 -1
- data/lib/karafka/cli/flow.rb +9 -6
- data/lib/karafka/cli/info.rb +1 -1
- data/lib/karafka/cli/install.rb +2 -0
- data/lib/karafka/connection/api_adapter.rb +7 -3
- data/lib/karafka/connection/builder.rb +4 -2
- data/lib/karafka/connection/client.rb +1 -1
- data/lib/karafka/connection/listener.rb +2 -2
- data/lib/karafka/contracts/consumer_group.rb +2 -2
- data/lib/karafka/contracts/server_cli_options.rb +2 -0
- data/lib/karafka/helpers/class_matcher.rb +1 -1
- data/lib/karafka/instrumentation/logger.rb +4 -3
- data/lib/karafka/server.rb +4 -1
- data/lib/karafka/version.rb +1 -1
- metadata +6 -6
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b4de7fea7ee892f9c83db91dea9e3d16d48d423985477d3f40440bea77907906
|
4
|
+
data.tar.gz: ccfe4744b3bb2c19b31032e8f904716116faa50f96a9be19a4e3cdc79364ebe6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 74dbf97aec22f4e0f5f9a07ac9b0457c09846332c65f548d30e2b0342a2adec4b8542004aa1937012892f3374b39f83f557ef5adecacc40ae538290842165150
|
7
|
+
data.tar.gz: fa2072670fe7c17720e593373a4d8222163f4e789c68ec72e4e1f58f4a08688839a0c01e98115db4dfbc5c57d633b0251fe481fc9838362203961c1e4e236159
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data.tar.gz.sig
CHANGED
Binary file
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.7.1
|
data/.travis.yml
CHANGED
@@ -12,25 +12,25 @@ test: &test
|
|
12
12
|
stage: Test
|
13
13
|
language: ruby
|
14
14
|
before_install:
|
15
|
-
- gem
|
16
|
-
- gem update --system
|
15
|
+
- yes | gem update --system
|
17
16
|
script: bundle exec rspec
|
18
17
|
|
19
18
|
jobs:
|
20
19
|
include:
|
21
20
|
- <<: *test
|
22
|
-
rvm: 2.
|
21
|
+
rvm: 2.7.1
|
23
22
|
- <<: *test
|
24
|
-
rvm: 2.
|
23
|
+
rvm: 2.6.6
|
24
|
+
- <<: *test
|
25
|
+
rvm: 2.5.8
|
25
26
|
|
26
27
|
- stage: coditsu
|
27
28
|
language: ruby
|
28
|
-
rvm: 2.
|
29
|
+
rvm: 2.7.1
|
29
30
|
before_install:
|
30
|
-
- gem update --system
|
31
|
-
- gem install bundler
|
31
|
+
- yes | gem update --system
|
32
32
|
script: \curl -sSL https://api.coditsu.io/run/ci | bash
|
33
33
|
|
34
34
|
stages:
|
35
|
-
- coditsu
|
36
35
|
- test
|
36
|
+
- coditsu
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,29 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
+
## 1.3.6 (2020-04-24)
|
4
|
+
- #583 - Use Karafka.logger for CLI messages (prikha)
|
5
|
+
- #582 - Cannot only define seed brokers in consumer groups
|
6
|
+
|
7
|
+
## 1.3.5 (2020-04-02)
|
8
|
+
- #578 - ThreadError: can't be called from trap context patch
|
9
|
+
|
10
|
+
## 1.3.4 (2020-02-17)
|
11
|
+
- `dry-configurable` upgrade (solnic)
|
12
|
+
- Remove temporary `thor` patches that are no longer needed
|
13
|
+
|
14
|
+
## 1.3.3 (2019-12-23)
|
15
|
+
- Require `delegate` to fix missing dependency in `ruby-kafka`
|
16
|
+
|
17
|
+
## 1.3.2 (2019-12-23)
|
18
|
+
- #561 - Allow `thor` 1.0.x usage in Karafka
|
19
|
+
- #567 - Ruby 2.7.0 support + unfreeze of a frozen string fix
|
20
|
+
|
21
|
+
## 1.3.1 (2019-11-11)
|
22
|
+
- #545 - Makes sure the log directory exists when is possible (robertomiranda)
|
23
|
+
- Ruby 2.6.5 support
|
24
|
+
- #551 - add support for DSA keys
|
25
|
+
- #549 - Missing directories after `karafka install` (nijikon)
|
26
|
+
|
3
27
|
## 1.3.0 (2019-09-09)
|
4
28
|
- Drop support for Ruby 2.4
|
5
29
|
- YARD docs tags cleanup
|
@@ -92,7 +116,7 @@
|
|
92
116
|
## 1.2.3
|
93
117
|
- #313 - support PLAINTEXT and SSL for scheme
|
94
118
|
- #288 - drop activesupport callbacks in favor of notifications
|
95
|
-
- #320 - Pausing
|
119
|
+
- #320 - Pausing indefinitely with nil pause timeout doesn't work
|
96
120
|
- #318 - Partition pausing doesn't work with custom topic mappers
|
97
121
|
- Rename ConfigAdapter to ApiAdapter to better reflect what it does
|
98
122
|
- #317 - Manual offset committing doesn't work with custom topic mappers
|
@@ -221,7 +245,7 @@
|
|
221
245
|
- Switch to multi json so everyone can use their favourite JSON parser
|
222
246
|
- Added jruby support in general and in Travis
|
223
247
|
- #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
|
224
|
-
- #96 - Karafka server -
|
248
|
+
- #96 - Karafka server - possibility to run it only for a certain topics
|
225
249
|
- ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
|
226
250
|
- (optional) pausing upon processing failures ```pause_timeout```
|
227
251
|
- Karafka console main process no longer intercepts irb errors
|
@@ -229,7 +253,7 @@
|
|
229
253
|
- #204 - Long running controllers
|
230
254
|
- Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
|
231
255
|
- #207 - Rename before_enqueued to after_received
|
232
|
-
- #147 -
|
256
|
+
- #147 - De-attach Karafka from Sidekiq by extracting Sidekiq backend
|
233
257
|
|
234
258
|
### New features and improvements
|
235
259
|
|
@@ -314,7 +338,7 @@
|
|
314
338
|
- Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
|
315
339
|
- #105 - Karafka::Monitor#caller_label not working with inherited monitors
|
316
340
|
- #99 - Standalone mode (without Sidekiq)
|
317
|
-
- #97 - Buffer responders single topics before send (
|
341
|
+
- #97 - Buffer responders single topics before send (pre-validation)
|
318
342
|
- Better control over consumer thanks to additional config options
|
319
343
|
- #111 - Dynamic worker assignment based on the income params
|
320
344
|
- Long shutdown time fix
|
@@ -322,7 +346,7 @@
|
|
322
346
|
## 0.5.0
|
323
347
|
- Removed Zookeeper totally as dependency
|
324
348
|
- Better group and partition rebalancing
|
325
|
-
- Automatic thread management (no need for
|
349
|
+
- Automatic thread management (no need for tuning) - each topic is a separate actor/thread
|
326
350
|
- Moved from Poseidon into Ruby-Kafka
|
327
351
|
- No more max_concurrency setting
|
328
352
|
- After you define your App class and routes (and everything else) you need to add execute App.boot!
|
@@ -338,14 +362,14 @@
|
|
338
362
|
- Ruby 2.2.* support dropped
|
339
363
|
- Using App name as a Kafka client_id
|
340
364
|
- Automatic Capistrano integration
|
341
|
-
- Responders support for handling better responses
|
365
|
+
- Responders support for handling better responses pipe-lining and better responses flow description and design (see README for more details)
|
342
366
|
- Gem bump
|
343
367
|
- Readme updates
|
344
368
|
- karafka flow CLI command for printing the application flow
|
345
|
-
- Some internal
|
369
|
+
- Some internal refactoring
|
346
370
|
|
347
371
|
## 0.4.2
|
348
|
-
- #87 -
|
372
|
+
- #87 - Re-consume mode with crone for better Rails/Rack integration
|
349
373
|
- Moved Karafka server related stuff into separate Karafka::Server class
|
350
374
|
- Renamed Karafka::Runner into Karafka::Fetcher
|
351
375
|
- Gem bump
|
@@ -357,7 +381,7 @@
|
|
357
381
|
|
358
382
|
## 0.4.1
|
359
383
|
- Explicit throw(:abort) required to halt before_enqueue (like in Rails 5)
|
360
|
-
- #61 -
|
384
|
+
- #61 - autodiscovery of Kafka brokers based on Zookeeper data
|
361
385
|
- #63 - Graceful shutdown with current offset state during data processing
|
362
386
|
- #65 - Example of NewRelic monitor is outdated
|
363
387
|
- #71 - Setup should be executed after user code is loaded
|
@@ -413,7 +437,7 @@
|
|
413
437
|
- Added Karafka::Monitoring that allows to add custom logging and monitoring with external libraries and systems
|
414
438
|
- Moved logging functionality into Karafka::Monitoring default monitoring
|
415
439
|
- Added possibility to provide own monitoring as long as in responds to #notice and #notice_error
|
416
|
-
-
|
440
|
+
- Standardized logging format for all logs
|
417
441
|
|
418
442
|
## 0.3.0
|
419
443
|
- Switched from custom ParserError for each parser to general catching of Karafka::Errors::ParseError and its descendants
|
@@ -430,7 +454,7 @@
|
|
430
454
|
|
431
455
|
## 0.1.19
|
432
456
|
- Internal call - schedule naming change
|
433
|
-
- Enqueue to perform_async naming in controller to follow
|
457
|
+
- Enqueue to perform_async naming in controller to follow Sidekiq naming convention
|
434
458
|
- Gem bump
|
435
459
|
|
436
460
|
## 0.1.18
|
@@ -441,7 +465,7 @@
|
|
441
465
|
- Changed Karafka::Connection::Cluster tp Karafka::Connection::ActorCluster to distinguish between a single thread actor cluster for multiple topic connection and a future feature that will allow process clusterization.
|
442
466
|
- Add an ability to use user-defined parsers for a messages
|
443
467
|
- Lazy load params for before callbacks
|
444
|
-
- Automatic loading/
|
468
|
+
- Automatic loading/initializing all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
|
445
469
|
- Params are now private to controller
|
446
470
|
- Added bootstrap method to app.rb
|
447
471
|
|
@@ -482,7 +506,7 @@
|
|
482
506
|
- Added worker logger
|
483
507
|
|
484
508
|
## 0.1.8
|
485
|
-
-
|
509
|
+
- Dropped local env support in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
|
486
510
|
|
487
511
|
## 0.1.7
|
488
512
|
- Karafka option for Redis hosts (not localhost only)
|
@@ -512,7 +536,7 @@
|
|
512
536
|
|
513
537
|
## 0.1.1
|
514
538
|
- README updates
|
515
|
-
-
|
539
|
+
- Rake tasks updates
|
516
540
|
- Rake installation task
|
517
541
|
- Changelog file added
|
518
542
|
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (1.3.
|
4
|
+
karafka (1.3.6)
|
5
5
|
dry-configurable (~> 0.8)
|
6
6
|
dry-inflector (~> 0.1)
|
7
7
|
dry-monitor (~> 0.3)
|
@@ -11,117 +11,120 @@ PATH
|
|
11
11
|
multi_json (>= 1.12)
|
12
12
|
rake (>= 11.3)
|
13
13
|
ruby-kafka (>= 0.7.8)
|
14
|
-
thor (
|
14
|
+
thor (>= 0.20)
|
15
15
|
waterdrop (~> 1.3.0)
|
16
16
|
zeitwerk (~> 2.1)
|
17
17
|
|
18
18
|
GEM
|
19
19
|
remote: https://rubygems.org/
|
20
20
|
specs:
|
21
|
-
activesupport (6.0.
|
21
|
+
activesupport (6.0.2.2)
|
22
22
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
23
23
|
i18n (>= 0.7, < 2)
|
24
24
|
minitest (~> 5.1)
|
25
25
|
tzinfo (~> 1.1)
|
26
|
-
zeitwerk (~> 2.
|
27
|
-
byebug (11.
|
28
|
-
concurrent-ruby (1.1.
|
29
|
-
delivery_boy (0.
|
26
|
+
zeitwerk (~> 2.2)
|
27
|
+
byebug (11.1.3)
|
28
|
+
concurrent-ruby (1.1.6)
|
29
|
+
delivery_boy (1.0.1)
|
30
30
|
king_konf (~> 0.3)
|
31
|
-
ruby-kafka (~> 0
|
31
|
+
ruby-kafka (~> 1.0)
|
32
32
|
diff-lcs (1.3)
|
33
|
-
digest-crc (0.
|
33
|
+
digest-crc (0.5.1)
|
34
34
|
docile (1.3.2)
|
35
|
-
dry-configurable (0.
|
35
|
+
dry-configurable (0.11.5)
|
36
36
|
concurrent-ruby (~> 1.0)
|
37
37
|
dry-core (~> 0.4, >= 0.4.7)
|
38
|
+
dry-equalizer (~> 0.2)
|
38
39
|
dry-container (0.7.2)
|
39
40
|
concurrent-ruby (~> 1.0)
|
40
41
|
dry-configurable (~> 0.1, >= 0.1.3)
|
41
42
|
dry-core (0.4.9)
|
42
43
|
concurrent-ruby (~> 1.0)
|
43
|
-
dry-equalizer (0.
|
44
|
+
dry-equalizer (0.3.0)
|
44
45
|
dry-events (0.2.0)
|
45
46
|
concurrent-ruby (~> 1.0)
|
46
47
|
dry-core (~> 0.4)
|
47
48
|
dry-equalizer (~> 0.2)
|
48
|
-
dry-inflector (0.
|
49
|
-
dry-initializer (3.0.
|
50
|
-
dry-logic (1.0.
|
49
|
+
dry-inflector (0.2.0)
|
50
|
+
dry-initializer (3.0.3)
|
51
|
+
dry-logic (1.0.6)
|
51
52
|
concurrent-ruby (~> 1.0)
|
52
53
|
dry-core (~> 0.2)
|
53
54
|
dry-equalizer (~> 0.2)
|
54
|
-
dry-monitor (0.3.
|
55
|
+
dry-monitor (0.3.2)
|
55
56
|
dry-configurable (~> 0.5)
|
56
57
|
dry-core (~> 0.4)
|
57
58
|
dry-equalizer (~> 0.2)
|
58
|
-
dry-events (~> 0.
|
59
|
-
dry-schema (1.
|
59
|
+
dry-events (~> 0.2)
|
60
|
+
dry-schema (1.5.0)
|
60
61
|
concurrent-ruby (~> 1.0)
|
61
62
|
dry-configurable (~> 0.8, >= 0.8.3)
|
62
63
|
dry-core (~> 0.4)
|
63
64
|
dry-equalizer (~> 0.2)
|
64
65
|
dry-initializer (~> 3.0)
|
65
66
|
dry-logic (~> 1.0)
|
66
|
-
dry-types (~> 1.
|
67
|
-
dry-types (1.
|
67
|
+
dry-types (~> 1.4)
|
68
|
+
dry-types (1.4.0)
|
68
69
|
concurrent-ruby (~> 1.0)
|
69
70
|
dry-container (~> 0.3)
|
70
71
|
dry-core (~> 0.4, >= 0.4.4)
|
71
|
-
dry-equalizer (~> 0.
|
72
|
+
dry-equalizer (~> 0.3)
|
72
73
|
dry-inflector (~> 0.1, >= 0.1.2)
|
73
74
|
dry-logic (~> 1.0, >= 1.0.2)
|
74
|
-
dry-validation (1.
|
75
|
+
dry-validation (1.5.0)
|
75
76
|
concurrent-ruby (~> 1.0)
|
76
77
|
dry-container (~> 0.7, >= 0.7.1)
|
77
78
|
dry-core (~> 0.4)
|
78
79
|
dry-equalizer (~> 0.2)
|
79
80
|
dry-initializer (~> 3.0)
|
80
|
-
dry-schema (~> 1.
|
81
|
-
envlogic (1.1.
|
81
|
+
dry-schema (~> 1.5)
|
82
|
+
envlogic (1.1.2)
|
82
83
|
dry-inflector (~> 0.1)
|
83
|
-
factory_bot (5.
|
84
|
+
factory_bot (5.1.2)
|
84
85
|
activesupport (>= 4.2.0)
|
85
|
-
i18n (1.
|
86
|
+
i18n (1.8.2)
|
86
87
|
concurrent-ruby (~> 1.0)
|
87
|
-
|
88
|
-
|
88
|
+
io-console (0.5.6)
|
89
|
+
irb (1.2.3)
|
90
|
+
reline (>= 0.0.1)
|
89
91
|
king_konf (0.3.7)
|
90
|
-
minitest (5.
|
91
|
-
multi_json (1.
|
92
|
-
rake (
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
rspec-
|
97
|
-
|
98
|
-
rspec-
|
99
|
-
rspec-
|
92
|
+
minitest (5.14.0)
|
93
|
+
multi_json (1.14.1)
|
94
|
+
rake (13.0.1)
|
95
|
+
reline (0.1.3)
|
96
|
+
io-console (~> 0.5)
|
97
|
+
rspec (3.9.0)
|
98
|
+
rspec-core (~> 3.9.0)
|
99
|
+
rspec-expectations (~> 3.9.0)
|
100
|
+
rspec-mocks (~> 3.9.0)
|
101
|
+
rspec-core (3.9.1)
|
102
|
+
rspec-support (~> 3.9.1)
|
103
|
+
rspec-expectations (3.9.1)
|
100
104
|
diff-lcs (>= 1.2.0, < 2.0)
|
101
|
-
rspec-support (~> 3.
|
102
|
-
rspec-mocks (3.
|
105
|
+
rspec-support (~> 3.9.0)
|
106
|
+
rspec-mocks (3.9.1)
|
103
107
|
diff-lcs (>= 1.2.0, < 2.0)
|
104
|
-
rspec-support (~> 3.
|
105
|
-
rspec-support (3.
|
106
|
-
ruby-kafka (0.
|
108
|
+
rspec-support (~> 3.9.0)
|
109
|
+
rspec-support (3.9.2)
|
110
|
+
ruby-kafka (1.0.0)
|
107
111
|
digest-crc
|
108
|
-
simplecov (0.
|
112
|
+
simplecov (0.18.5)
|
109
113
|
docile (~> 1.1)
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
thor (0.20.3)
|
114
|
+
simplecov-html (~> 0.11)
|
115
|
+
simplecov-html (0.12.2)
|
116
|
+
thor (1.0.1)
|
114
117
|
thread_safe (0.3.6)
|
115
|
-
tzinfo (1.2.
|
118
|
+
tzinfo (1.2.7)
|
116
119
|
thread_safe (~> 0.1)
|
117
|
-
waterdrop (1.3.
|
118
|
-
delivery_boy (
|
120
|
+
waterdrop (1.3.4)
|
121
|
+
delivery_boy (>= 0.2, < 2.x)
|
119
122
|
dry-configurable (~> 0.8)
|
120
123
|
dry-monitor (~> 0.3)
|
121
124
|
dry-validation (~> 1.2)
|
122
125
|
ruby-kafka (>= 0.7.8)
|
123
126
|
zeitwerk (~> 2.1)
|
124
|
-
zeitwerk (2.
|
127
|
+
zeitwerk (2.3.0)
|
125
128
|
|
126
129
|
PLATFORMS
|
127
130
|
ruby
|
@@ -134,4 +137,4 @@ DEPENDENCIES
|
|
134
137
|
simplecov
|
135
138
|
|
136
139
|
BUNDLED WITH
|
137
|
-
2.
|
140
|
+
2.1.4
|
data/README.md
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
[](https://travis-ci.org/karafka/karafka)
|
4
4
|
|
5
|
-
|
5
|
+
**Note**: Documentation presented here refers to Karafka `1.3.x`.
|
6
6
|
|
7
|
-
|
7
|
+
If you're upgrading from `1.2.0`, please refer to our [Upgrade Notes article](https://mensfeld.pl/2019/09/karafka-framework-1-3-0-release-notes-ruby-kafka/).
|
8
8
|
|
9
|
-
If you are looking for the documentation for Karafka `1.2
|
9
|
+
If you are looking for the documentation for Karafka `1.2.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
|
10
10
|
|
11
11
|
## About Karafka
|
12
12
|
|
data/karafka.gemspec
CHANGED
@@ -26,11 +26,11 @@ Gem::Specification.new do |spec|
|
|
26
26
|
spec.add_dependency 'multi_json', '>= 1.12'
|
27
27
|
spec.add_dependency 'rake', '>= 11.3'
|
28
28
|
spec.add_dependency 'ruby-kafka', '>= 0.7.8'
|
29
|
-
spec.add_dependency 'thor', '
|
29
|
+
spec.add_dependency 'thor', '>= 0.20'
|
30
30
|
spec.add_dependency 'waterdrop', '~> 1.3.0'
|
31
31
|
spec.add_dependency 'zeitwerk', '~> 2.1'
|
32
32
|
|
33
|
-
spec.required_ruby_version = '>= 2.
|
33
|
+
spec.required_ruby_version = '>= 2.5.0'
|
34
34
|
|
35
35
|
if $PROGRAM_NAME.end_with?('gem')
|
36
36
|
spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
|
data/lib/karafka.rb
CHANGED
@@ -52,14 +52,8 @@ module Karafka
|
|
52
52
|
ignored_settings = api_adapter[:subscribe]
|
53
53
|
defined_settings = api_adapter.values.flatten
|
54
54
|
karafka_settings = %i[batch_fetching]
|
55
|
-
|
56
|
-
dynamically_proxied = Karafka::Setup::Config
|
57
|
-
._settings
|
58
|
-
.settings
|
59
|
-
.find { |s| s.name == :kafka }
|
60
|
-
.value
|
61
|
-
.names
|
62
|
-
.to_a
|
55
|
+
|
56
|
+
dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
|
63
57
|
|
64
58
|
(defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
|
65
59
|
end
|
data/lib/karafka/cli.rb
CHANGED
data/lib/karafka/cli/flow.rb
CHANGED
@@ -11,19 +11,22 @@ module Karafka
|
|
11
11
|
def call
|
12
12
|
topics.each do |topic|
|
13
13
|
any_topics = !topic.responder&.topics.nil?
|
14
|
+
log_messages = []
|
14
15
|
|
15
16
|
if any_topics
|
16
|
-
|
17
|
+
log_messages << "#{topic.name} =>"
|
17
18
|
|
18
19
|
topic.responder.topics.each_value do |responder_topic|
|
19
20
|
features = []
|
20
21
|
features << (responder_topic.required? ? 'always' : 'conditionally')
|
21
22
|
|
22
|
-
|
23
|
+
log_messages << format(responder_topic.name, "(#{features.join(', ')})")
|
23
24
|
end
|
24
25
|
else
|
25
|
-
|
26
|
+
log_messages << "#{topic.name} => (nothing)"
|
26
27
|
end
|
28
|
+
|
29
|
+
Karafka.logger.info(log_messages.join("\n"))
|
27
30
|
end
|
28
31
|
end
|
29
32
|
|
@@ -34,11 +37,11 @@ module Karafka
|
|
34
37
|
Karafka::App.consumer_groups.map(&:topics).flatten.sort_by(&:name)
|
35
38
|
end
|
36
39
|
|
37
|
-
#
|
40
|
+
# Formats a given value with label in a nice way
|
38
41
|
# @param label [String] label describing value
|
39
42
|
# @param value [String] value that should be printed
|
40
|
-
def
|
41
|
-
|
43
|
+
def format(label, value)
|
44
|
+
" - #{label}: #{value}"
|
42
45
|
end
|
43
46
|
end
|
44
47
|
end
|
data/lib/karafka/cli/info.rb
CHANGED
data/lib/karafka/cli/install.rb
CHANGED
@@ -14,11 +14,12 @@ module Karafka
|
|
14
14
|
module ApiAdapter
|
15
15
|
class << self
|
16
16
|
# Builds all the configuration settings for Kafka.new method
|
17
|
+
# @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
|
17
18
|
# @return [Array<Hash>] Array with all the client arguments including hash with all
|
18
19
|
# the settings required by Kafka.new method
|
19
20
|
# @note We return array, so we can inject any arguments we want, in case of changes in the
|
20
21
|
# raw driver
|
21
|
-
def client
|
22
|
+
def client(consumer_group)
|
22
23
|
# This one is a default that takes all the settings except special
|
23
24
|
# cases defined in the map
|
24
25
|
settings = {
|
@@ -26,14 +27,17 @@ module Karafka
|
|
26
27
|
client_id: ::Karafka::App.config.client_id
|
27
28
|
}
|
28
29
|
|
29
|
-
kafka_configs.
|
30
|
+
kafka_configs.each_key do |setting_name|
|
30
31
|
# All options for config adapter should be ignored as we're just interested
|
31
32
|
# in what is left, as we want to pass all the options that are "typical"
|
32
33
|
# and not listed in the api_adapter special cases mapping. All the values
|
33
34
|
# from the api_adapter mapping go somewhere else, not to the client directly
|
34
35
|
next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
|
35
36
|
|
36
|
-
|
37
|
+
# Settings for each consumer group are either defined per consumer group or are
|
38
|
+
# inherited from the global/general settings level, thus we don't have to fetch them
|
39
|
+
# from the kafka settings as they are already on a consumer group level
|
40
|
+
settings[setting_name] = consumer_group.public_send(setting_name)
|
37
41
|
end
|
38
42
|
|
39
43
|
settings_hash = sanitize(settings)
|
@@ -6,9 +6,11 @@ module Karafka
|
|
6
6
|
module Builder
|
7
7
|
class << self
|
8
8
|
# Builds a Kafka::Client instance that we use to work with Kafka cluster
|
9
|
+
# @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which we want
|
10
|
+
# to have a new Kafka client
|
9
11
|
# @return [::Kafka::Client] returns a Kafka client
|
10
|
-
def call
|
11
|
-
Kafka.new(*ApiAdapter.client)
|
12
|
+
def call(consumer_group)
|
13
|
+
Kafka.new(*ApiAdapter.client(consumer_group))
|
12
14
|
end
|
13
15
|
end
|
14
16
|
end
|
@@ -97,7 +97,7 @@ module Karafka
|
|
97
97
|
def kafka_consumer
|
98
98
|
# @note We don't cache the connection internally because we cache kafka_consumer that uses
|
99
99
|
# kafka client object instance
|
100
|
-
@kafka_consumer ||= Builder.call.consumer(
|
100
|
+
@kafka_consumer ||= Builder.call(consumer_group).consumer(
|
101
101
|
*ApiAdapter.consumer(consumer_group)
|
102
102
|
).tap do |consumer|
|
103
103
|
consumer_group.topics.each do |topic|
|
@@ -47,10 +47,10 @@ module Karafka
|
|
47
47
|
end
|
48
48
|
end
|
49
49
|
# This is on purpose - see the notes for this method
|
50
|
-
# rubocop:disable RescueException
|
50
|
+
# rubocop:disable Lint/RescueException
|
51
51
|
rescue Exception => e
|
52
52
|
Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
|
53
|
-
# rubocop:enable RescueException
|
53
|
+
# rubocop:enable Lint/RescueException
|
54
54
|
# We can stop client without a problem, as it will reinitialize itself when running the
|
55
55
|
# `fetch_loop` again
|
56
56
|
@client.stop
|
@@ -178,9 +178,9 @@ module Karafka
|
|
178
178
|
# @param value [String] potential RSA key value
|
179
179
|
# @return [Boolean] is the given string a valid RSA key
|
180
180
|
def valid_private_key?(value)
|
181
|
-
OpenSSL::PKey
|
181
|
+
OpenSSL::PKey.read(value)
|
182
182
|
true
|
183
|
-
rescue OpenSSL::PKey::
|
183
|
+
rescue OpenSSL::PKey::PKeyError
|
184
184
|
false
|
185
185
|
end
|
186
186
|
|
@@ -6,6 +6,8 @@ module Karafka
|
|
6
6
|
# We validate some basics + the list of consumer_groups on which we want to use, to make
|
7
7
|
# sure that all of them are defined, plus that a pidfile does not exist
|
8
8
|
class ServerCliOptions < Dry::Validation::Contract
|
9
|
+
config.messages.load_paths << File.join(Karafka.gem_root, 'config', 'errors.yml')
|
10
|
+
|
9
11
|
params do
|
10
12
|
optional(:pid).filled(:str?)
|
11
13
|
optional(:daemon).filled(:bool?)
|
@@ -44,7 +44,7 @@ module Karafka
|
|
44
44
|
# @example From Namespaced::Super2Consumer matching responder
|
45
45
|
# matcher.name #=> Super2Responder
|
46
46
|
def name
|
47
|
-
inflected =
|
47
|
+
inflected = +@klass.to_s.split('::').last.to_s
|
48
48
|
# We inject the from into the name just in case it is missing as in a situation like
|
49
49
|
# that it would just sanitize the name without adding the "to" postfix.
|
50
50
|
# It could create cases when we want to build for example a responder to a consumer
|
@@ -36,10 +36,11 @@ module Karafka
|
|
36
36
|
.to(STDOUT, file)
|
37
37
|
end
|
38
38
|
|
39
|
-
# Makes sure the log directory exists
|
39
|
+
# Makes sure the log directory exists as long as we can write to it
|
40
40
|
def ensure_dir_exists
|
41
|
-
|
42
|
-
|
41
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
42
|
+
rescue Errno::EACCES
|
43
|
+
nil
|
43
44
|
end
|
44
45
|
|
45
46
|
# @return [Pathname] Path to a file to which we should log
|
data/lib/karafka/server.rb
CHANGED
@@ -58,9 +58,12 @@ module Karafka
|
|
58
58
|
def stop_supervised
|
59
59
|
Karafka::App.stop!
|
60
60
|
|
61
|
+
# Temporary patch until https://github.com/dry-rb/dry-configurable/issues/93 is fixed
|
62
|
+
timeout = Thread.new { Karafka::App.config.shutdown_timeout }.join.value
|
63
|
+
|
61
64
|
# We check from time to time (for the timeout period) if all the threads finished
|
62
65
|
# their work and if so, we can just return and normal shutdown process will take place
|
63
|
-
(
|
66
|
+
(timeout * SUPERVISION_CHECK_FACTOR).to_i.times do
|
64
67
|
if consumer_threads.count(&:alive?).zero?
|
65
68
|
Thread.new { Karafka.monitor.instrument('app.stopped') }.join
|
66
69
|
return
|
data/lib/karafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
|
37
37
|
nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date:
|
39
|
+
date: 2020-04-24 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: dry-configurable
|
@@ -168,14 +168,14 @@ dependencies:
|
|
168
168
|
name: thor
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
170
170
|
requirements:
|
171
|
-
- - "
|
171
|
+
- - ">="
|
172
172
|
- !ruby/object:Gem::Version
|
173
173
|
version: '0.20'
|
174
174
|
type: :runtime
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
|
-
- - "
|
178
|
+
- - ">="
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '0.20'
|
181
181
|
- !ruby/object:Gem::Dependency
|
@@ -322,14 +322,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
322
322
|
requirements:
|
323
323
|
- - ">="
|
324
324
|
- !ruby/object:Gem::Version
|
325
|
-
version: 2.
|
325
|
+
version: 2.5.0
|
326
326
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
327
327
|
requirements:
|
328
328
|
- - ">="
|
329
329
|
- !ruby/object:Gem::Version
|
330
330
|
version: '0'
|
331
331
|
requirements: []
|
332
|
-
rubygems_version: 3.
|
332
|
+
rubygems_version: 3.1.2
|
333
333
|
signing_key:
|
334
334
|
specification_version: 4
|
335
335
|
summary: Ruby based framework for working with Apache Kafka
|
metadata.gz.sig
CHANGED
Binary file
|