karafka 1.3.0 → 1.4.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.diffend.yml +3 -0
- data/.github/workflows/ci.yml +76 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +112 -15
- data/CODE_OF_CONDUCT.md +1 -1
- data/Gemfile +2 -0
- data/Gemfile.lock +87 -98
- data/README.md +28 -31
- data/certs/mensfeld.pem +24 -23
- data/config/errors.yml +2 -0
- data/docker-compose.yml +17 -0
- data/karafka.gemspec +22 -14
- data/lib/karafka/assignment_strategies/round_robin.rb +13 -0
- data/lib/karafka/attributes_map.rb +3 -8
- data/lib/karafka/cli/base.rb +4 -4
- data/lib/karafka/cli/flow.rb +9 -6
- data/lib/karafka/cli/info.rb +1 -1
- data/lib/karafka/cli/install.rb +5 -2
- data/lib/karafka/cli/missingno.rb +19 -0
- data/lib/karafka/cli/server.rb +8 -8
- data/lib/karafka/cli.rb +9 -1
- data/lib/karafka/connection/api_adapter.rb +27 -24
- data/lib/karafka/connection/batch_delegator.rb +5 -1
- data/lib/karafka/connection/builder.rb +9 -2
- data/lib/karafka/connection/client.rb +9 -6
- data/lib/karafka/connection/listener.rb +2 -2
- data/lib/karafka/consumers/batch_metadata.rb +10 -0
- data/lib/karafka/consumers/includer.rb +5 -4
- data/lib/karafka/contracts/consumer_group.rb +10 -5
- data/lib/karafka/contracts/server_cli_options.rb +2 -0
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/helpers/class_matcher.rb +2 -2
- data/lib/karafka/instrumentation/logger.rb +6 -9
- data/lib/karafka/instrumentation/stdout_listener.rb +6 -4
- data/lib/karafka/params/batch_metadata.rb +26 -0
- data/lib/karafka/params/builders/batch_metadata.rb +30 -0
- data/lib/karafka/params/builders/params.rb +17 -15
- data/lib/karafka/params/builders/params_batch.rb +2 -2
- data/lib/karafka/params/metadata.rb +14 -29
- data/lib/karafka/params/params.rb +27 -41
- data/lib/karafka/params/params_batch.rb +15 -16
- data/lib/karafka/routing/builder.rb +1 -0
- data/lib/karafka/routing/consumer_group.rb +5 -3
- data/lib/karafka/serialization/json/deserializer.rb +2 -2
- data/lib/karafka/server.rb +4 -1
- data/lib/karafka/setup/config.rb +60 -52
- data/lib/karafka/templates/karafka.rb.erb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +3 -1
- data.tar.gz.sig +0 -0
- metadata +75 -93
- metadata.gz.sig +0 -0
- data/.github/FUNDING.yml +0 -3
- data/.travis.yml +0 -36
- data/lib/karafka/consumers/metadata.rb +0 -10
- data/lib/karafka/params/builders/metadata.rb +0 -33
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7dd06a7ace623ae63695899e2cff1293482390ccbaeabcf7b1cc4b4aa6ec6a9e
|
|
4
|
+
data.tar.gz: 60e7c986a94c9552c1adc754b6bdb02f5e5cb5012881a15333fa8eff854485a8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cc34ba15cd7f8f138202fd0a9b53c3f63fcde13dd353e8da810a2b1f5e153c87335b0f4d26e6dccca4c484ddc12ef59d8331315130772bd5f05ef39a34f1a7c7
|
|
7
|
+
data.tar.gz: 9ef7cfce8c382091072e1c1df4cfece7bd5220cc3b8c74f82f4b1166bda756c354b4632df66ef95c912d3f13c2264914d96359fb72d4663592f98cd4b313f269
|
checksums.yaml.gz.sig
CHANGED
|
Binary file
|
data/.diffend.yml
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
concurrency: ci-${{ github.ref }}
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
pull_request:
|
|
7
|
+
push:
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: '0 1 * * *'
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
specs:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
needs: diffend
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
ruby:
|
|
19
|
+
- '3.1'
|
|
20
|
+
- '3.0'
|
|
21
|
+
- '2.7'
|
|
22
|
+
include:
|
|
23
|
+
- ruby: '3.1'
|
|
24
|
+
coverage: 'true'
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v2
|
|
27
|
+
- name: Install package dependencies
|
|
28
|
+
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
|
29
|
+
- name: Set up Ruby
|
|
30
|
+
uses: ruby/setup-ruby@v1
|
|
31
|
+
with:
|
|
32
|
+
ruby-version: ${{matrix.ruby}}
|
|
33
|
+
- name: Install latest bundler
|
|
34
|
+
run: |
|
|
35
|
+
gem install bundler --no-document
|
|
36
|
+
bundle config set without 'tools benchmarks docs'
|
|
37
|
+
- name: Bundle install
|
|
38
|
+
run: |
|
|
39
|
+
bundle config set without development
|
|
40
|
+
bundle install --jobs 4 --retry 3
|
|
41
|
+
- name: Run Kafka with docker-compose
|
|
42
|
+
run: docker-compose up -d
|
|
43
|
+
- name: Run all tests
|
|
44
|
+
env:
|
|
45
|
+
GITHUB_COVERAGE: ${{matrix.coverage}}
|
|
46
|
+
run: bundle exec rspec
|
|
47
|
+
|
|
48
|
+
diffend:
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
strategy:
|
|
51
|
+
fail-fast: false
|
|
52
|
+
steps:
|
|
53
|
+
- uses: actions/checkout@v2
|
|
54
|
+
with:
|
|
55
|
+
fetch-depth: 0
|
|
56
|
+
- name: Set up Ruby
|
|
57
|
+
uses: ruby/setup-ruby@v1
|
|
58
|
+
with:
|
|
59
|
+
ruby-version: 3.1
|
|
60
|
+
- name: Install latest bundler
|
|
61
|
+
run: gem install bundler --no-document
|
|
62
|
+
- name: Install Diffend plugin
|
|
63
|
+
run: bundle plugin install diffend
|
|
64
|
+
- name: Bundle Secure
|
|
65
|
+
run: bundle secure
|
|
66
|
+
|
|
67
|
+
coditsu:
|
|
68
|
+
runs-on: ubuntu-latest
|
|
69
|
+
strategy:
|
|
70
|
+
fail-fast: false
|
|
71
|
+
steps:
|
|
72
|
+
- uses: actions/checkout@v2
|
|
73
|
+
with:
|
|
74
|
+
fetch-depth: 0
|
|
75
|
+
- name: Run Coditsu
|
|
76
|
+
run: \curl -sSL https://api.coditsu.io/run/ci | bash
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
3.1.2
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,102 @@
|
|
|
1
1
|
# Karafka framework changelog
|
|
2
2
|
|
|
3
|
+
## 1.4.14 (2022-10-14)
|
|
4
|
+
- Fix `concurrent-ruby` missing as a dependency (Azdaroth)
|
|
5
|
+
- Warn about upcoming end of 1.4 support.
|
|
6
|
+
|
|
7
|
+
## 1.4.13 (2022-02-19)
|
|
8
|
+
- Drop support for ruby 2.6
|
|
9
|
+
- Add mfa requirement
|
|
10
|
+
|
|
11
|
+
## 1.4.12 (2022-01-13)
|
|
12
|
+
- Ruby 3.1 support
|
|
13
|
+
- `irb` dependency removal (vbyno)
|
|
14
|
+
|
|
15
|
+
## 1.4.11 (2021-12-04)
|
|
16
|
+
- Source code metadata url added to the gemspec
|
|
17
|
+
- Gem bump
|
|
18
|
+
|
|
19
|
+
## 1.4.10 (2021-10-30)
|
|
20
|
+
- update gems requirements in the gemspec (nijikon)
|
|
21
|
+
|
|
22
|
+
## 1.4.9 (2021-09-29)
|
|
23
|
+
- fix `dry-configurable` deprecation warnings for default value as positional argument
|
|
24
|
+
|
|
25
|
+
## 1.4.8 (2021-09-08)
|
|
26
|
+
- Allow 'rails' in Gemfile to enable rails-aware generator (rewritten)
|
|
27
|
+
|
|
28
|
+
## 1.4.7 (2021-09-04)
|
|
29
|
+
- Update ruby-kafka to `1.4.0`
|
|
30
|
+
- Support for `resolve_seed_brokers` option (with Azdaroth)
|
|
31
|
+
- Set minimum `ruby-kafka` requirement to `1.3.0`
|
|
32
|
+
|
|
33
|
+
## 1.4.6 (2021-08-05)
|
|
34
|
+
- #700 Fix Ruby 3 compatibility issues in Connection::Client#pause (MmKolodziej)
|
|
35
|
+
|
|
36
|
+
## 1.4.5 (2021-06-16)
|
|
37
|
+
- Fixup logger checks for non-writeable logfile (ojab)
|
|
38
|
+
- #689 - Update the stdout initialization message for framework initialization
|
|
39
|
+
|
|
40
|
+
## 1.4.4 (2021-04-19)
|
|
41
|
+
- Remove Ruby 2.5 support and update minimum Ruby requirement to 2.6
|
|
42
|
+
- Remove rake dependency
|
|
43
|
+
|
|
44
|
+
## 1.4.3 (2021-03-24)
|
|
45
|
+
- Fixes for Ruby 3.0 compatibility
|
|
46
|
+
|
|
47
|
+
## 1.4.2 (2021-02-16)
|
|
48
|
+
- Rescue Errno::EROFS in ensure_dir_exists (unasuke)
|
|
49
|
+
|
|
50
|
+
## 1.4.1 (2020-12-04)
|
|
51
|
+
- Return non-zero exit code when printing usage
|
|
52
|
+
- Add support for :assignment_strategy for consumers
|
|
53
|
+
|
|
54
|
+
## 1.4.0 (2020-09-05)
|
|
55
|
+
- Rename `Karafka::Params::Metadata` to `Karafka::Params::BatchMetadata`
|
|
56
|
+
- Rename consumer `#metadata` to `#batch_metadata`
|
|
57
|
+
- Separate metadata (including Karafka native metadata) from the root of params (backwards compatibility preserved thanks to rabotyaga)
|
|
58
|
+
- Remove metadata hash dependency
|
|
59
|
+
- Remove params dependency on a hash in favour of PORO
|
|
60
|
+
- Remove batch metadata dependency on a hash
|
|
61
|
+
- Remove MultiJson in favour of JSON in the default deserializer
|
|
62
|
+
- allow accessing all the metadata without accessing the payload
|
|
63
|
+
- freeze params and underlying elements except for the mutable payload
|
|
64
|
+
- provide access to raw payload after serialization
|
|
65
|
+
- fixes a bug where a non-deserializable (error) params would be marked as deserialized after first unsuccessful deserialization attempt
|
|
66
|
+
- fixes bug where karafka would mutate internal ruby-kafka state
|
|
67
|
+
- fixes bug where topic name in metadata would not be mapped using topic mappers
|
|
68
|
+
- simplifies the params and params batch API, before `#payload` usage, it won't be deserialized
|
|
69
|
+
- removes the `#[]` API from params to prevent from accessing raw data in a different way than #raw_payload
|
|
70
|
+
- makes the params batch operations consistent as params payload is deserialized only when accessed explicitly
|
|
71
|
+
|
|
72
|
+
## 1.3.7 (2020-08-11)
|
|
73
|
+
- #599 - Allow metadata access without deserialization attempt (rabotyaga)
|
|
74
|
+
- Sync with ruby-kafka `1.2.0` api
|
|
75
|
+
|
|
76
|
+
## 1.3.6 (2020-04-24)
|
|
77
|
+
- #583 - Use Karafka.logger for CLI messages (prikha)
|
|
78
|
+
- #582 - Cannot only define seed brokers in consumer groups
|
|
79
|
+
|
|
80
|
+
## 1.3.5 (2020-04-02)
|
|
81
|
+
- #578 - ThreadError: can't be called from trap context patch
|
|
82
|
+
|
|
83
|
+
## 1.3.4 (2020-02-17)
|
|
84
|
+
- `dry-configurable` upgrade (solnic)
|
|
85
|
+
- Remove temporary `thor` patches that are no longer needed
|
|
86
|
+
|
|
87
|
+
## 1.3.3 (2019-12-23)
|
|
88
|
+
- Require `delegate` to fix missing dependency in `ruby-kafka`
|
|
89
|
+
|
|
90
|
+
## 1.3.2 (2019-12-23)
|
|
91
|
+
- #561 - Allow `thor` 1.0.x usage in Karafka
|
|
92
|
+
- #567 - Ruby 2.7.0 support + unfreeze of a frozen string fix
|
|
93
|
+
|
|
94
|
+
## 1.3.1 (2019-11-11)
|
|
95
|
+
- #545 - Makes sure the log directory exists when is possible (robertomiranda)
|
|
96
|
+
- Ruby 2.6.5 support
|
|
97
|
+
- #551 - add support for DSA keys
|
|
98
|
+
- #549 - Missing directories after `karafka install` (nijikon)
|
|
99
|
+
|
|
3
100
|
## 1.3.0 (2019-09-09)
|
|
4
101
|
- Drop support for Ruby 2.4
|
|
5
102
|
- YARD docs tags cleanup
|
|
@@ -58,6 +155,7 @@
|
|
|
58
155
|
- #508 - Reset the consumers instances upon reconnecting to a cluster
|
|
59
156
|
- [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
|
|
60
157
|
- [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
|
|
158
|
+
- [#319](https://github.com/karafka/karafka/pull/328) - Support for exponential backoff in pause
|
|
61
159
|
|
|
62
160
|
## 1.2.11
|
|
63
161
|
- [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
|
|
@@ -92,11 +190,10 @@
|
|
|
92
190
|
## 1.2.3
|
|
93
191
|
- #313 - support PLAINTEXT and SSL for scheme
|
|
94
192
|
- #288 - drop activesupport callbacks in favor of notifications
|
|
95
|
-
- #320 - Pausing
|
|
193
|
+
- #320 - Pausing indefinitely with nil pause timeout doesn't work
|
|
96
194
|
- #318 - Partition pausing doesn't work with custom topic mappers
|
|
97
195
|
- Rename ConfigAdapter to ApiAdapter to better reflect what it does
|
|
98
196
|
- #317 - Manual offset committing doesn't work with custom topic mappers
|
|
99
|
-
- #319 - Support for exponential backoff in pause
|
|
100
197
|
|
|
101
198
|
## 1.2.2
|
|
102
199
|
- #312 - Broken for ActiveSupport 5.2.0
|
|
@@ -221,7 +318,7 @@
|
|
|
221
318
|
- Switch to multi json so everyone can use their favourite JSON parser
|
|
222
319
|
- Added jruby support in general and in Travis
|
|
223
320
|
- #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
|
|
224
|
-
- #96 - Karafka server -
|
|
321
|
+
- #96 - Karafka server - possibility to run it only for a certain topics
|
|
225
322
|
- ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
|
|
226
323
|
- (optional) pausing upon processing failures ```pause_timeout```
|
|
227
324
|
- Karafka console main process no longer intercepts irb errors
|
|
@@ -229,7 +326,7 @@
|
|
|
229
326
|
- #204 - Long running controllers
|
|
230
327
|
- Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
|
|
231
328
|
- #207 - Rename before_enqueued to after_received
|
|
232
|
-
- #147 -
|
|
329
|
+
- #147 - De-attach Karafka from Sidekiq by extracting Sidekiq backend
|
|
233
330
|
|
|
234
331
|
### New features and improvements
|
|
235
332
|
|
|
@@ -314,7 +411,7 @@
|
|
|
314
411
|
- Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
|
|
315
412
|
- #105 - Karafka::Monitor#caller_label not working with inherited monitors
|
|
316
413
|
- #99 - Standalone mode (without Sidekiq)
|
|
317
|
-
- #97 - Buffer responders single topics before send (
|
|
414
|
+
- #97 - Buffer responders single topics before send (pre-validation)
|
|
318
415
|
- Better control over consumer thanks to additional config options
|
|
319
416
|
- #111 - Dynamic worker assignment based on the income params
|
|
320
417
|
- Long shutdown time fix
|
|
@@ -322,7 +419,7 @@
|
|
|
322
419
|
## 0.5.0
|
|
323
420
|
- Removed Zookeeper totally as dependency
|
|
324
421
|
- Better group and partition rebalancing
|
|
325
|
-
- Automatic thread management (no need for
|
|
422
|
+
- Automatic thread management (no need for tuning) - each topic is a separate actor/thread
|
|
326
423
|
- Moved from Poseidon into Ruby-Kafka
|
|
327
424
|
- No more max_concurrency setting
|
|
328
425
|
- After you define your App class and routes (and everything else) you need to add execute App.boot!
|
|
@@ -338,14 +435,14 @@
|
|
|
338
435
|
- Ruby 2.2.* support dropped
|
|
339
436
|
- Using App name as a Kafka client_id
|
|
340
437
|
- Automatic Capistrano integration
|
|
341
|
-
- Responders support for handling better responses
|
|
438
|
+
- Responders support for handling better responses pipe-lining and better responses flow description and design (see README for more details)
|
|
342
439
|
- Gem bump
|
|
343
440
|
- Readme updates
|
|
344
441
|
- karafka flow CLI command for printing the application flow
|
|
345
|
-
- Some internal
|
|
442
|
+
- Some internal refactoring
|
|
346
443
|
|
|
347
444
|
## 0.4.2
|
|
348
|
-
- #87 -
|
|
445
|
+
- #87 - Re-consume mode with crone for better Rails/Rack integration
|
|
349
446
|
- Moved Karafka server related stuff into separate Karafka::Server class
|
|
350
447
|
- Renamed Karafka::Runner into Karafka::Fetcher
|
|
351
448
|
- Gem bump
|
|
@@ -357,7 +454,7 @@
|
|
|
357
454
|
|
|
358
455
|
## 0.4.1
|
|
359
456
|
- Explicit throw(:abort) required to halt before_enqueue (like in Rails 5)
|
|
360
|
-
- #61 -
|
|
457
|
+
- #61 - autodiscovery of Kafka brokers based on Zookeeper data
|
|
361
458
|
- #63 - Graceful shutdown with current offset state during data processing
|
|
362
459
|
- #65 - Example of NewRelic monitor is outdated
|
|
363
460
|
- #71 - Setup should be executed after user code is loaded
|
|
@@ -413,7 +510,7 @@
|
|
|
413
510
|
- Added Karafka::Monitoring that allows to add custom logging and monitoring with external libraries and systems
|
|
414
511
|
- Moved logging functionality into Karafka::Monitoring default monitoring
|
|
415
512
|
- Added possibility to provide own monitoring as long as in responds to #notice and #notice_error
|
|
416
|
-
-
|
|
513
|
+
- Standardized logging format for all logs
|
|
417
514
|
|
|
418
515
|
## 0.3.0
|
|
419
516
|
- Switched from custom ParserError for each parser to general catching of Karafka::Errors::ParseError and its descendants
|
|
@@ -430,7 +527,7 @@
|
|
|
430
527
|
|
|
431
528
|
## 0.1.19
|
|
432
529
|
- Internal call - schedule naming change
|
|
433
|
-
- Enqueue to perform_async naming in controller to follow
|
|
530
|
+
- Enqueue to perform_async naming in controller to follow Sidekiq naming convention
|
|
434
531
|
- Gem bump
|
|
435
532
|
|
|
436
533
|
## 0.1.18
|
|
@@ -441,7 +538,7 @@
|
|
|
441
538
|
- Changed Karafka::Connection::Cluster tp Karafka::Connection::ActorCluster to distinguish between a single thread actor cluster for multiple topic connection and a future feature that will allow process clusterization.
|
|
442
539
|
- Add an ability to use user-defined parsers for a messages
|
|
443
540
|
- Lazy load params for before callbacks
|
|
444
|
-
- Automatic loading/
|
|
541
|
+
- Automatic loading/initializing all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
|
|
445
542
|
- Params are now private to controller
|
|
446
543
|
- Added bootstrap method to app.rb
|
|
447
544
|
|
|
@@ -482,7 +579,7 @@
|
|
|
482
579
|
- Added worker logger
|
|
483
580
|
|
|
484
581
|
## 0.1.8
|
|
485
|
-
-
|
|
582
|
+
- Dropped local env support in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
|
|
486
583
|
|
|
487
584
|
## 0.1.7
|
|
488
585
|
- Karafka option for Redis hosts (not localhost only)
|
|
@@ -512,7 +609,7 @@
|
|
|
512
609
|
|
|
513
610
|
## 0.1.1
|
|
514
611
|
- README updates
|
|
515
|
-
-
|
|
612
|
+
- Rake tasks updates
|
|
516
613
|
- Rake installation task
|
|
517
614
|
- Changelog file added
|
|
518
615
|
|
data/CODE_OF_CONDUCT.md
CHANGED
|
@@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe
|
|
|
34
34
|
|
|
35
35
|
## Enforcement
|
|
36
36
|
|
|
37
|
-
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@
|
|
37
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maciej@mensfeld.pl. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
|
|
38
38
|
|
|
39
39
|
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
|
|
40
40
|
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,130 +1,119 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
karafka (1.
|
|
5
|
-
|
|
6
|
-
dry-
|
|
7
|
-
dry-
|
|
8
|
-
dry-
|
|
4
|
+
karafka (1.4.14)
|
|
5
|
+
concurrent-ruby
|
|
6
|
+
dry-configurable (~> 0.16)
|
|
7
|
+
dry-inflector (~> 0.2)
|
|
8
|
+
dry-monitor (~> 0.5)
|
|
9
|
+
dry-validation (~> 1.7)
|
|
9
10
|
envlogic (~> 1.1)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
thor (~> 0.20)
|
|
15
|
-
waterdrop (~> 1.3.0)
|
|
16
|
-
zeitwerk (~> 2.1)
|
|
11
|
+
ruby-kafka (>= 1.3.0)
|
|
12
|
+
thor (>= 1.1)
|
|
13
|
+
waterdrop (~> 1.4)
|
|
14
|
+
zeitwerk (~> 2.6)
|
|
17
15
|
|
|
18
16
|
GEM
|
|
19
17
|
remote: https://rubygems.org/
|
|
20
18
|
specs:
|
|
21
|
-
activesupport (
|
|
19
|
+
activesupport (7.0.3)
|
|
22
20
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
23
|
-
i18n (>=
|
|
24
|
-
minitest (
|
|
25
|
-
tzinfo (~>
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
docile (1.
|
|
35
|
-
dry-configurable (0.
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
dry-container (0.
|
|
21
|
+
i18n (>= 1.6, < 2)
|
|
22
|
+
minitest (>= 5.1)
|
|
23
|
+
tzinfo (~> 2.0)
|
|
24
|
+
byebug (11.1.3)
|
|
25
|
+
concurrent-ruby (1.1.10)
|
|
26
|
+
delivery_boy (1.1.0)
|
|
27
|
+
king_konf (~> 1.0)
|
|
28
|
+
ruby-kafka (~> 1.0)
|
|
29
|
+
diff-lcs (1.5.0)
|
|
30
|
+
digest-crc (0.6.4)
|
|
31
|
+
rake (>= 12.0.0, < 14.0.0)
|
|
32
|
+
docile (1.4.0)
|
|
33
|
+
dry-configurable (0.16.1)
|
|
34
|
+
dry-core (~> 0.6)
|
|
35
|
+
zeitwerk (~> 2.6)
|
|
36
|
+
dry-container (0.11.0)
|
|
39
37
|
concurrent-ruby (~> 1.0)
|
|
40
|
-
|
|
41
|
-
dry-core (0.4.9)
|
|
38
|
+
dry-core (0.8.1)
|
|
42
39
|
concurrent-ruby (~> 1.0)
|
|
43
|
-
dry-
|
|
44
|
-
dry-events (0.2.0)
|
|
40
|
+
dry-events (0.3.0)
|
|
45
41
|
concurrent-ruby (~> 1.0)
|
|
46
|
-
dry-core (~> 0.
|
|
47
|
-
|
|
48
|
-
dry-
|
|
49
|
-
dry-
|
|
50
|
-
dry-logic (1.0.3)
|
|
42
|
+
dry-core (~> 0.5, >= 0.5)
|
|
43
|
+
dry-inflector (0.3.0)
|
|
44
|
+
dry-initializer (3.1.1)
|
|
45
|
+
dry-logic (1.2.0)
|
|
51
46
|
concurrent-ruby (~> 1.0)
|
|
52
|
-
dry-core (~> 0.
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
dry-
|
|
56
|
-
dry-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
dry-schema (1.3.3)
|
|
47
|
+
dry-core (~> 0.5, >= 0.5)
|
|
48
|
+
dry-monitor (0.6.3)
|
|
49
|
+
dry-configurable (~> 0.13, >= 0.13.0)
|
|
50
|
+
dry-core (~> 0.5, >= 0.5)
|
|
51
|
+
dry-events (~> 0.2)
|
|
52
|
+
zeitwerk (~> 2.5)
|
|
53
|
+
dry-schema (1.10.6)
|
|
60
54
|
concurrent-ruby (~> 1.0)
|
|
61
|
-
dry-configurable (~> 0.
|
|
62
|
-
dry-core (~> 0.
|
|
63
|
-
dry-equalizer (~> 0.2)
|
|
55
|
+
dry-configurable (~> 0.13, >= 0.13.0)
|
|
56
|
+
dry-core (~> 0.5, >= 0.5)
|
|
64
57
|
dry-initializer (~> 3.0)
|
|
65
|
-
dry-logic (~> 1.
|
|
66
|
-
dry-types (~> 1.
|
|
67
|
-
dry-types (1.
|
|
58
|
+
dry-logic (~> 1.2)
|
|
59
|
+
dry-types (~> 1.5)
|
|
60
|
+
dry-types (1.5.1)
|
|
68
61
|
concurrent-ruby (~> 1.0)
|
|
69
62
|
dry-container (~> 0.3)
|
|
70
|
-
dry-core (~> 0.
|
|
71
|
-
dry-equalizer (~> 0.2, >= 0.2.2)
|
|
63
|
+
dry-core (~> 0.5, >= 0.5)
|
|
72
64
|
dry-inflector (~> 0.1, >= 0.1.2)
|
|
73
65
|
dry-logic (~> 1.0, >= 1.0.2)
|
|
74
|
-
dry-validation (1.
|
|
66
|
+
dry-validation (1.8.1)
|
|
75
67
|
concurrent-ruby (~> 1.0)
|
|
76
68
|
dry-container (~> 0.7, >= 0.7.1)
|
|
77
|
-
dry-core (~> 0.
|
|
78
|
-
dry-equalizer (~> 0.2)
|
|
69
|
+
dry-core (~> 0.5, >= 0.5)
|
|
79
70
|
dry-initializer (~> 3.0)
|
|
80
|
-
dry-schema (~> 1.
|
|
81
|
-
envlogic (1.1.
|
|
71
|
+
dry-schema (~> 1.8, >= 1.8.0)
|
|
72
|
+
envlogic (1.1.5)
|
|
82
73
|
dry-inflector (~> 0.1)
|
|
83
|
-
factory_bot (
|
|
84
|
-
activesupport (>=
|
|
85
|
-
i18n (1.
|
|
74
|
+
factory_bot (6.2.1)
|
|
75
|
+
activesupport (>= 5.0.0)
|
|
76
|
+
i18n (1.10.0)
|
|
86
77
|
concurrent-ruby (~> 1.0)
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
rspec-
|
|
96
|
-
|
|
97
|
-
rspec-core (3.8.2)
|
|
98
|
-
rspec-support (~> 3.8.0)
|
|
99
|
-
rspec-expectations (3.8.4)
|
|
78
|
+
king_konf (1.0.1)
|
|
79
|
+
minitest (5.15.0)
|
|
80
|
+
rake (13.0.6)
|
|
81
|
+
rspec (3.11.0)
|
|
82
|
+
rspec-core (~> 3.11.0)
|
|
83
|
+
rspec-expectations (~> 3.11.0)
|
|
84
|
+
rspec-mocks (~> 3.11.0)
|
|
85
|
+
rspec-core (3.11.0)
|
|
86
|
+
rspec-support (~> 3.11.0)
|
|
87
|
+
rspec-expectations (3.11.0)
|
|
100
88
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
101
|
-
rspec-support (~> 3.
|
|
102
|
-
rspec-mocks (3.
|
|
89
|
+
rspec-support (~> 3.11.0)
|
|
90
|
+
rspec-mocks (3.11.1)
|
|
103
91
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
104
|
-
rspec-support (~> 3.
|
|
105
|
-
rspec-support (3.
|
|
106
|
-
ruby-kafka (
|
|
92
|
+
rspec-support (~> 3.11.0)
|
|
93
|
+
rspec-support (3.11.0)
|
|
94
|
+
ruby-kafka (1.5.0)
|
|
107
95
|
digest-crc
|
|
108
|
-
simplecov (0.
|
|
96
|
+
simplecov (0.21.2)
|
|
109
97
|
docile (~> 1.1)
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
simplecov-html (0.
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
tzinfo (
|
|
116
|
-
|
|
117
|
-
waterdrop (1.
|
|
118
|
-
delivery_boy (
|
|
119
|
-
dry-configurable (~> 0.
|
|
120
|
-
dry-monitor (~> 0.
|
|
121
|
-
dry-validation (~> 1.
|
|
122
|
-
ruby-kafka (>=
|
|
123
|
-
zeitwerk (~> 2.
|
|
124
|
-
zeitwerk (2.1
|
|
98
|
+
simplecov-html (~> 0.11)
|
|
99
|
+
simplecov_json_formatter (~> 0.1)
|
|
100
|
+
simplecov-html (0.12.3)
|
|
101
|
+
simplecov_json_formatter (0.1.4)
|
|
102
|
+
thor (1.2.1)
|
|
103
|
+
tzinfo (2.0.4)
|
|
104
|
+
concurrent-ruby (~> 1.0)
|
|
105
|
+
waterdrop (1.4.4)
|
|
106
|
+
delivery_boy (>= 0.2, < 2.x)
|
|
107
|
+
dry-configurable (~> 0.13)
|
|
108
|
+
dry-monitor (~> 0.5)
|
|
109
|
+
dry-validation (~> 1.7)
|
|
110
|
+
ruby-kafka (>= 1.3.0)
|
|
111
|
+
zeitwerk (~> 2.4)
|
|
112
|
+
zeitwerk (2.6.1)
|
|
125
113
|
|
|
126
114
|
PLATFORMS
|
|
127
|
-
|
|
115
|
+
x86_64-darwin-18
|
|
116
|
+
x86_64-linux
|
|
128
117
|
|
|
129
118
|
DEPENDENCIES
|
|
130
119
|
byebug
|
|
@@ -134,4 +123,4 @@ DEPENDENCIES
|
|
|
134
123
|
simplecov
|
|
135
124
|
|
|
136
125
|
BUNDLED WITH
|
|
137
|
-
2.
|
|
126
|
+
2.3.11
|
data/README.md
CHANGED
|
@@ -1,17 +1,33 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
|
-
[](https://github.com/karafka/karafka/actions/workflows/ci.yml)
|
|
4
|
+
[](http://badge.fury.io/rb/karafka)
|
|
5
|
+
[](https://slack.karafka.io)
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
**Note**: Documentation presented here refers to Karafka `1.3.0`.
|
|
8
|
-
|
|
9
|
-
If you are looking for the documentation for Karafka `1.2.*`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
|
|
7
|
+
**Note**: We're finishing the new Karafka `2.0` but for now, please use `1.4`. All the documentation presented here refers to `1.4`
|
|
10
8
|
|
|
11
9
|
## About Karafka
|
|
12
10
|
|
|
13
11
|
Framework used to simplify Apache Kafka based Ruby applications development.
|
|
14
12
|
|
|
13
|
+
```ruby
|
|
14
|
+
# Define what topics you want to consume with which consumers
|
|
15
|
+
Karafka::App.consumer_groups.draw do
|
|
16
|
+
topic 'system_events' do
|
|
17
|
+
consumer EventsConsumer
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# And create your consumers, within which your messages will be processed
|
|
22
|
+
class EventsConsumer < ApplicationConsumer
|
|
23
|
+
# Example that utilizes ActiveRecord#insert_all and Karafka batch processing
|
|
24
|
+
def consume
|
|
25
|
+
# Store all of the incoming Kafka events locally in an efficient way
|
|
26
|
+
Event.insert_all params_batch.payloads
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
```
|
|
30
|
+
|
|
15
31
|
Karafka allows you to capture everything that happens in your systems in large scale, providing you with a seamless and stable core for consuming and processing this data, without having to focus on things that are not your business domain.
|
|
16
32
|
|
|
17
33
|
Karafka not only handles incoming messages but also provides tools for building complex data-flow applications that receive and send messages.
|
|
@@ -37,9 +53,9 @@ Karafka based applications can be easily deployed to any type of infrastructure,
|
|
|
37
53
|
|
|
38
54
|
## Support
|
|
39
55
|
|
|
40
|
-
Karafka has
|
|
56
|
+
Karafka has [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup, and deployment along with other useful details on how to run Karafka.
|
|
41
57
|
|
|
42
|
-
If you have any questions about using Karafka, feel free to join our [
|
|
58
|
+
If you have any questions about using Karafka, feel free to join our [Slack](https://slack.karafka.io) chat channel.
|
|
43
59
|
|
|
44
60
|
## Getting started
|
|
45
61
|
|
|
@@ -67,35 +83,16 @@ and follow the instructions from the [example app Wiki](https://github.com/karaf
|
|
|
67
83
|
|
|
68
84
|
If you need more details and know how on how to start Karafka with a clean installation, read the [Getting started page](https://github.com/karafka/karafka/wiki/Getting-started) section of our Wiki.
|
|
69
85
|
|
|
70
|
-
## Notice
|
|
71
|
-
|
|
72
|
-
Karafka framework and Karafka team are __not__ related to Kafka streaming service called CloudKarafka in any matter. We don't recommend nor discourage usage of their platform.
|
|
73
|
-
|
|
74
86
|
## References
|
|
75
87
|
|
|
76
88
|
* [Karafka framework](https://github.com/karafka/karafka)
|
|
77
|
-
* [Karafka
|
|
89
|
+
* [Karafka GitHub Actions](https://github.com/karafka/karafka/actions)
|
|
78
90
|
* [Karafka Coditsu](https://app.coditsu.io/karafka/repositories/karafka)
|
|
79
91
|
|
|
80
92
|
## Note on contributions
|
|
81
93
|
|
|
82
|
-
First, thank you for considering contributing to Karafka! It's people like you that make the open source community such a great community!
|
|
83
|
-
|
|
84
|
-
Each pull request must pass all the RSpec specs and meet our quality requirements.
|
|
85
|
-
|
|
86
|
-
To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
|
|
87
|
-
|
|
88
|
-
Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/commit_builds) of Karafka organization.
|
|
89
|
-
|
|
90
|
-
[](https://app.coditsu.io/karafka/commit_builds)
|
|
91
|
-
|
|
92
|
-
## Contributors
|
|
93
|
-
|
|
94
|
-
This project exists thanks to all the people who contribute.
|
|
95
|
-
<a href="https://github.com/karafka/karafka/graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>
|
|
96
|
-
|
|
97
|
-
## Sponsors
|
|
94
|
+
First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!
|
|
98
95
|
|
|
99
|
-
|
|
96
|
+
Each pull request must pass all the RSpec specs, integration tests and meet our quality requirements.
|
|
100
97
|
|
|
101
|
-
|
|
98
|
+
Fork it, update and wait for the Github Actions results.
|