karafka 1.3.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +1 -0
  4. data/.coditsu/ci.yml +3 -0
  5. data/.console_irbrc +11 -0
  6. data/.github/FUNDING.yml +3 -0
  7. data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
  8. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  9. data/.gitignore +69 -0
  10. data/.rspec +1 -0
  11. data/.ruby-gemset +1 -0
  12. data/.ruby-version +1 -0
  13. data/.travis.yml +36 -0
  14. data/CHANGELOG.md +544 -0
  15. data/CODE_OF_CONDUCT.md +46 -0
  16. data/CONTRIBUTING.md +41 -0
  17. data/Gemfile +12 -0
  18. data/Gemfile.lock +140 -0
  19. data/MIT-LICENCE +18 -0
  20. data/README.md +101 -0
  21. data/bin/karafka +19 -0
  22. data/certs/mensfeld.pem +25 -0
  23. data/config/errors.yml +39 -0
  24. data/karafka.gemspec +44 -0
  25. data/lib/karafka.rb +72 -0
  26. data/lib/karafka/app.rb +53 -0
  27. data/lib/karafka/attributes_map.rb +62 -0
  28. data/lib/karafka/backends/inline.rb +16 -0
  29. data/lib/karafka/base_consumer.rb +57 -0
  30. data/lib/karafka/base_responder.rb +226 -0
  31. data/lib/karafka/cli.rb +54 -0
  32. data/lib/karafka/cli/base.rb +78 -0
  33. data/lib/karafka/cli/console.rb +31 -0
  34. data/lib/karafka/cli/flow.rb +48 -0
  35. data/lib/karafka/cli/info.rb +31 -0
  36. data/lib/karafka/cli/install.rb +66 -0
  37. data/lib/karafka/cli/server.rb +71 -0
  38. data/lib/karafka/code_reloader.rb +67 -0
  39. data/lib/karafka/connection/api_adapter.rb +159 -0
  40. data/lib/karafka/connection/batch_delegator.rb +51 -0
  41. data/lib/karafka/connection/builder.rb +18 -0
  42. data/lib/karafka/connection/client.rb +117 -0
  43. data/lib/karafka/connection/listener.rb +71 -0
  44. data/lib/karafka/connection/message_delegator.rb +36 -0
  45. data/lib/karafka/consumers/callbacks.rb +71 -0
  46. data/lib/karafka/consumers/includer.rb +63 -0
  47. data/lib/karafka/consumers/metadata.rb +10 -0
  48. data/lib/karafka/consumers/responders.rb +24 -0
  49. data/lib/karafka/consumers/single_params.rb +15 -0
  50. data/lib/karafka/contracts.rb +10 -0
  51. data/lib/karafka/contracts/config.rb +21 -0
  52. data/lib/karafka/contracts/consumer_group.rb +206 -0
  53. data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
  54. data/lib/karafka/contracts/responder_usage.rb +54 -0
  55. data/lib/karafka/contracts/server_cli_options.rb +31 -0
  56. data/lib/karafka/errors.rb +51 -0
  57. data/lib/karafka/fetcher.rb +42 -0
  58. data/lib/karafka/helpers/class_matcher.rb +88 -0
  59. data/lib/karafka/helpers/config_retriever.rb +46 -0
  60. data/lib/karafka/helpers/inflector.rb +26 -0
  61. data/lib/karafka/helpers/multi_delegator.rb +32 -0
  62. data/lib/karafka/instrumentation/logger.rb +58 -0
  63. data/lib/karafka/instrumentation/monitor.rb +70 -0
  64. data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
  65. data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
  66. data/lib/karafka/params/builders/metadata.rb +33 -0
  67. data/lib/karafka/params/builders/params.rb +36 -0
  68. data/lib/karafka/params/builders/params_batch.rb +25 -0
  69. data/lib/karafka/params/metadata.rb +35 -0
  70. data/lib/karafka/params/params.rb +68 -0
  71. data/lib/karafka/params/params_batch.rb +61 -0
  72. data/lib/karafka/patches/ruby_kafka.rb +47 -0
  73. data/lib/karafka/persistence/client.rb +29 -0
  74. data/lib/karafka/persistence/consumers.rb +45 -0
  75. data/lib/karafka/persistence/topics.rb +48 -0
  76. data/lib/karafka/process.rb +60 -0
  77. data/lib/karafka/responders/builder.rb +36 -0
  78. data/lib/karafka/responders/topic.rb +55 -0
  79. data/lib/karafka/routing/builder.rb +89 -0
  80. data/lib/karafka/routing/consumer_group.rb +61 -0
  81. data/lib/karafka/routing/consumer_mapper.rb +34 -0
  82. data/lib/karafka/routing/proxy.rb +46 -0
  83. data/lib/karafka/routing/router.rb +29 -0
  84. data/lib/karafka/routing/topic.rb +62 -0
  85. data/lib/karafka/routing/topic_mapper.rb +53 -0
  86. data/lib/karafka/serialization/json/deserializer.rb +27 -0
  87. data/lib/karafka/serialization/json/serializer.rb +31 -0
  88. data/lib/karafka/server.rb +86 -0
  89. data/lib/karafka/setup/config.rb +221 -0
  90. data/lib/karafka/setup/configurators/water_drop.rb +36 -0
  91. data/lib/karafka/setup/dsl.rb +21 -0
  92. data/lib/karafka/status.rb +29 -0
  93. data/lib/karafka/templates/application_consumer.rb.erb +7 -0
  94. data/lib/karafka/templates/application_responder.rb.erb +11 -0
  95. data/lib/karafka/templates/karafka.rb.erb +92 -0
  96. data/lib/karafka/version.rb +7 -0
  97. data/log/.gitkeep +0 -0
  98. metadata +336 -0
  99. metadata.gz.sig +3 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: b4de7fea7ee892f9c83db91dea9e3d16d48d423985477d3f40440bea77907906
4
+ data.tar.gz: ccfe4744b3bb2c19b31032e8f904716116faa50f96a9be19a4e3cdc79364ebe6
5
+ SHA512:
6
+ metadata.gz: 74dbf97aec22f4e0f5f9a07ac9b0457c09846332c65f548d30e2b0342a2adec4b8542004aa1937012892f3374b39f83f557ef5adecacc40ae538290842165150
7
+ data.tar.gz: fa2072670fe7c17720e593373a4d8222163f4e789c68ec72e4e1f58f4a08688839a0c01e98115db4dfbc5c57d633b0251fe481fc9838362203961c1e4e236159
Binary file
@@ -0,0 +1 @@
1
+ �*�AK����do>�Gb�;�N��3$,�b��j�,uJ�=�2ZI/]��_ɣ����|�3k��x���s�Z�ʋ�`
@@ -0,0 +1,3 @@
1
+ repository_id: 'd4482d42-f6b5-44ba-a5e4-00989ac519ee'
2
+ api_key: <%= ENV['CODITSU_API_KEY'] %>
3
+ api_secret: <%= ENV['CODITSU_API_SECRET'] %>
@@ -0,0 +1,11 @@
1
+ # irbrc for Karafka console
2
+
3
+ IRB.conf[:AUTO_INDENT] = true
4
+ IRB.conf[:SAVE_HISTORY] = 1000
5
+ IRB.conf[:USE_READLINE] = true
6
+ IRB.conf[:HISTORY_FILE] = ".irb-history"
7
+ IRB.conf[:LOAD_MODULES] = [] unless IRB.conf.key?(:LOAD_MODULES)
8
+
9
+ unless IRB.conf[:LOAD_MODULES].include?('irb/completion')
10
+ IRB.conf[:LOAD_MODULES] << 'irb/completion'
11
+ end
@@ -0,0 +1,3 @@
1
+ # These are supported funding model platforms
2
+
3
+ open_collective: karafka
@@ -0,0 +1,50 @@
1
+ ---
2
+ name: Bug Report
3
+ about: Report an issue with Karafka you've discovered.
4
+ ---
5
+
6
+ *Be clear, concise and precise in your description of the problem.
7
+ Open an issue with a descriptive title and a summary in grammatically correct,
8
+ complete sentences.*
9
+
10
+ *Use the template below when reporting bugs. Please, make sure that
11
+ you're running the latest stable Karafka and that the problem you're reporting
12
+ hasn't been reported (and potentially fixed) already.*
13
+
14
+ *Before filing the ticket you should replace all text above the horizontal
15
+ rule with your own words.*
16
+
17
+ --------
18
+
19
+ ## Expected behavior
20
+
21
+ Describe here how you expected Karafka to behave in this particular situation.
22
+
23
+ ## Actual behavior
24
+
25
+ Describe here what actually happened.
26
+
27
+ ## Steps to reproduce the problem
28
+
29
+ This is extremely important! Providing us with a reliable way to reproduce
30
+ a problem will expedite its solution.
31
+
32
+ ## Your setup details
33
+
34
+ Please provide kafka version and the output of `karafka info` or `bundle exec karafka info` if using Bundler.
35
+
36
+ Here's an example:
37
+
38
+ ```
39
+ $ [bundle exec] karafka info
40
+ Karafka version: 1.3.0
41
+ Ruby version: 2.6.3
42
+ Ruby-kafka version: 0.7.9
43
+ Application client id: karafka-local
44
+ Backend: inline
45
+ Batch fetching: true
46
+ Batch consuming: true
47
+ Boot file: /app/karafka/karafka.rb
48
+ Environment: development
49
+ Kafka seed brokers: ["kafka://kafka:9092"]
50
+ ```
@@ -0,0 +1,20 @@
1
+ ---
2
+ name: Feature Request
3
+ about: Suggest new Karafka features or improvements to existing features.
4
+ ---
5
+
6
+ ## Is your feature request related to a problem? Please describe.
7
+
8
+ A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
9
+
10
+ ## Describe the solution you'd like
11
+
12
+ A clear and concise description of what you want to happen.
13
+
14
+ ## Describe alternatives you've considered
15
+
16
+ A clear and concise description of any alternative solutions or features you've considered.
17
+
18
+ ## Additional context
19
+
20
+ Add any other context or screenshots about the feature request here.
@@ -0,0 +1,69 @@
1
+ # bundler state
2
+ /.bundle
3
+ /vendor/bundle/
4
+ /vendor/ruby/
5
+ /ruby/
6
+ app.god
7
+
8
+ # minimal Rails specific artifacts
9
+ /.coditsu/local.yml
10
+ db/*.sqlite3
11
+ /log/development.log
12
+ /log/production.log
13
+ /log/test.log
14
+ /tmp/*
15
+ *.gem
16
+ *.~
17
+
18
+ # various artifacts
19
+ **.war
20
+ *.rbc
21
+ *.sassc
22
+ .byebug_history
23
+ .redcar/
24
+ .capistrano/
25
+ .sass-cache
26
+ /config/god/sidekiq.rb
27
+ /config/puma.rb
28
+ /coverage.data
29
+ /coverage/
30
+ /doc/api/
31
+ /doc/app/
32
+ /doc/yard
33
+ /doc/features.html
34
+ /doc/specs.html
35
+ /spec/tmp/*
36
+ /cache
37
+ /capybara*
38
+ /capybara-*.html
39
+ /gems
40
+ /specifications
41
+ rerun.txt
42
+ pickle-email-*.html
43
+
44
+ # If you find yourself ignoring temporary files generated by your text editor
45
+ # or operating system, you probably want to add a global ignore instead:
46
+ # git config --global core.excludesfile ~/.gitignore_global
47
+ #
48
+ # Here are some files you may want to ignore globally:
49
+
50
+ # scm revert files
51
+ **.orig
52
+
53
+ # Mac finder artifacts
54
+ .DS_Store
55
+
56
+ # Netbeans project directory
57
+ /nbproject
58
+
59
+ # RubyMine project files
60
+ .idea
61
+
62
+ # Textmate project files
63
+ /*.tmproj
64
+
65
+ # vim artifacts
66
+ **.swp
67
+
68
+ # documentation
69
+ .yardoc
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --require spec_helper
@@ -0,0 +1 @@
1
+ karafka
@@ -0,0 +1 @@
1
+ 2.7.1
@@ -0,0 +1,36 @@
1
+ services:
2
+ - docker
3
+
4
+ dist: trusty
5
+ sudo: false
6
+ cache: bundler
7
+
8
+ git:
9
+ depth: false
10
+
11
+ test: &test
12
+ stage: Test
13
+ language: ruby
14
+ before_install:
15
+ - yes | gem update --system
16
+ script: bundle exec rspec
17
+
18
+ jobs:
19
+ include:
20
+ - <<: *test
21
+ rvm: 2.7.1
22
+ - <<: *test
23
+ rvm: 2.6.6
24
+ - <<: *test
25
+ rvm: 2.5.8
26
+
27
+ - stage: coditsu
28
+ language: ruby
29
+ rvm: 2.7.1
30
+ before_install:
31
+ - yes | gem update --system
32
+ script: \curl -sSL https://api.coditsu.io/run/ci | bash
33
+
34
+ stages:
35
+ - test
36
+ - coditsu
@@ -0,0 +1,544 @@
1
+ # Karafka framework changelog
2
+
3
+ ## 1.3.6 (2020-04-24)
4
+ - #583 - Use Karafka.logger for CLI messages (prikha)
5
+ - #582 - Cannot only define seed brokers in consumer groups
6
+
7
+ ## 1.3.5 (2020-04-02)
8
+ - #578 - ThreadError: can't be called from trap context patch
9
+
10
+ ## 1.3.4 (2020-02-17)
11
+ - `dry-configurable` upgrade (solnic)
12
+ - Remove temporary `thor` patches that are no longer needed
13
+
14
+ ## 1.3.3 (2019-12-23)
15
+ - Require `delegate` to fix missing dependency in `ruby-kafka`
16
+
17
+ ## 1.3.2 (2019-12-23)
18
+ - #561 - Allow `thor` 1.0.x usage in Karafka
19
+ - #567 - Ruby 2.7.0 support + unfreeze of a frozen string fix
20
+
21
+ ## 1.3.1 (2019-11-11)
22
+ - #545 - Makes sure the log directory exists when is possible (robertomiranda)
23
+ - Ruby 2.6.5 support
24
+ - #551 - add support for DSA keys
25
+ - #549 - Missing directories after `karafka install` (nijikon)
26
+
27
+ ## 1.3.0 (2019-09-09)
28
+ - Drop support for Ruby 2.4
29
+ - YARD docs tags cleanup
30
+
31
+ ## 1.3.0.rc1 (2019-07-31)
32
+ - Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
33
+ - Update ruby-kafka to the 0.7 version
34
+ - Support messages headers receiving
35
+ - Message bus unification
36
+ - Parser available in metadata
37
+ - Cleanup towards moving to a non-global state app management
38
+ - Drop Ruby 2.3 support
39
+ - Support for Ruby 2.6.3
40
+ - `Karafka::Loader` has been removed in favor of Zeitwerk
41
+ - Schemas are now contracts
42
+ - #393 - Reorganize responders - removed `multiple_usage` constrain
43
+ - #388 - ssl_client_cert_chain sync
44
+ - #300 - Store value in a value key and replace its content with parsed version - without root merge
45
+ - #331 - Disallow building groups without topics
46
+ - #340 - Instrumentation unification. Better and more consistent naming
47
+ - #340 - Procline instrumentation for a nicer process name
48
+ - #342 - Change default for `fetcher_max_queue_size` from `100` to `10` to lower max memory usage
49
+ - #345 - Cleanup exceptions names
50
+ - #341 - Split connection delegator into batch delegator and single_delegator
51
+ - #351 - Rename `#retrieve!` to `#parse!` on params and `#parsed` to `parse!` on params batch.
52
+ - #351 - Adds '#first' for params_batch that returns parsed first element from the params_batch object.
53
+ - #360 - Single params consuming mode automatically parses data specs
54
+ - #359 - Divide mark_as_consumed into mark_as_consumed and mark_as_consumed!
55
+ - #356 - Provide a `#values` for params_batch to extract only values of objects from the params_batch
56
+ - #363 - Too shallow ruby-kafka version lock
57
+ - #354 - Expose consumer heartbeat
58
+ - #377 - Remove the persistent setup in favor of persistence
59
+ - #375 - Sidekiq Backend parser mismatch
60
+ - #369 - Single consumer can support more than one topic
61
+ - #288 - Drop dependency on `activesupport` gem
62
+ - #371 - SASL over SSL
63
+ - #392 - Move params redundant data to metadata
64
+ - #335 - Metadata access from within the consumer
65
+ - #402 - Delayed reconnection upon critical failures
66
+ - #405 - `reconnect_timeout` value is now being validated
67
+ - #437 - Specs ensuring that the `#437` won't occur in the `1.3` release
68
+ - #426 - ssl client cert key password
69
+ - #444 - add certificate and private key validation
70
+ - #460 - Decouple responder "parser" (generator?) from topic.parser (benissimo)
71
+ - #463 - Split parsers into serializers / deserializers
72
+ - #473 - Support SASL OAuthBearer Authentication
73
+ - #475 - Disallow subscribing to the same topic with multiple consumers
74
+ - #485 - Setting shutdown_timeout to nil kills the app without waiting for anything
75
+ - #487 - Make listeners as instances
76
+ - #29 - Consumer class names must have the word "Consumer" in it in order to work (Sidekiq backend)
77
+ - #491 - irb is missing for console to work
78
+ - #502 - Karafka process hangs when sending multiple sigkills
79
+ - #506 - ssl_verify_hostname sync
80
+ - #483 - Upgrade dry-validation before releasing 1.3
81
+ - #492 - Use Zeitwerk for code reload in development
82
+ - #508 - Reset the consumers instances upon reconnecting to a cluster
83
+ - [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
84
+ - [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
85
+
86
+ ## 1.2.11
87
+ - [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
88
+
89
+ ## 1.2.10
90
+ - [#453](https://github.com/karafka/karafka/pull/453) require `Forwardable` module
91
+
92
+ ## 1.2.9
93
+ - Critical exceptions now will cause consumer to stop instead of retrying without a break
94
+ - #412 - Fix dry-inflector dependency lock in gemspec
95
+ - #414 - Backport to 1.2 the delayed retry upon failure
96
+ - #437 - Raw message is no longer added to params after ParserError raised
97
+
98
+ ## 1.2.8
99
+ - #408 - Responder Topic Lookup Bug on Heroku
100
+
101
+ ## 1.2.7
102
+ - Unlock Ruby-kafka version with a warning
103
+
104
+ ## 1.2.6
105
+ - Lock WaterDrop to 1.2.3
106
+ - Lock Ruby-Kafka to 0.6.x (support for 0.7 will be added in Karafka 1.3)
107
+ - #382 - Full logging with AR, etc for development mode when there is Rails integration
108
+
109
+ ## 1.2.5
110
+ - #354 - Expose consumer heartbeat
111
+ - #373 - Async producer not working properly with responders
112
+
113
+ ## 1.2.4
114
+ - #332 - Fetcher for max queue size
115
+
116
+ ## 1.2.3
117
+ - #313 - support PLAINTEXT and SSL for scheme
118
+ - #288 - drop activesupport callbacks in favor of notifications
119
+ - #320 - Pausing indefinitely with nil pause timeout doesn't work
120
+ - #318 - Partition pausing doesn't work with custom topic mappers
121
+ - Rename ConfigAdapter to ApiAdapter to better reflect what it does
122
+ - #317 - Manual offset committing doesn't work with custom topic mappers
123
+ - #319 - Support for exponential backoff in pause
124
+
125
+ ## 1.2.2
126
+ - #312 - Broken for ActiveSupport 5.2.0
127
+
128
+ ## 1.2.1
129
+ - #304 - Unification of error instrumentation event details
130
+ - #306 - Using file logger from within a trap context upon shutdown is impossible
131
+
132
+ ## 1.2.0
133
+ - Spec improvements
134
+ - #260 - Specs missing randomization
135
+ - #251 - Shutdown upon non responding (unreachable) cluster is not possible
136
+ - #258 - Investigate lowering requirements on activesupport
137
+ - #246 - Alias consumer#mark_as_consumed on controller
138
+ - #259 - Allow forcing key/partition key on responders
139
+ - #267 - Styling inconsistency
140
+ - #242 - Support setting the max bytes to fetch per request
141
+ - #247 - Support SCRAM once released
142
+ - #271 - Provide an after_init option to pass a configuration block
143
+ - #262 - Error in the monitor code for NewRelic
144
+ - #241 - Performance metrics
145
+ - #274 - Rename controllers to consumers
146
+ - #184 - Seek to
147
+ - #284 - Dynamic Params parent class
148
+ - #275 - ssl_ca_certs_from_system
149
+ - #296 - Instrument forceful exit with an error
150
+ - Replaced some of the activesupport parts with dry-inflector
151
+ - Lower ActiveSupport dependency
152
+ - Remove configurators in favor of the after_init block configurator
153
+ - Ruby 2.5.0 support
154
+ - Renamed Karafka::Connection::Processor to Karafka::Connection::Delegator to match incoming naming conventions
155
+ - Renamed Karafka::Connection::Consumer to Karafka::Connection::Client due to #274
156
+ - Removed HashWithIndifferentAccess in favor of a regular hash
157
+ - JSON parsing defaults now to string keys
158
+ - Lower memory usage due to less params data internal details
159
+ - Support multiple ```after_init``` blocks in favor of a single one
160
+ - Renamed ```received_at``` to ```receive_time``` to follow ruby-kafka and WaterDrop conventions
161
+ - Adjust internal setup to easier map Ruby-Kafka config changes
162
+ - System callbacks reorganization
163
+ - Added ```before_fetch_loop``` configuration block for early client usage (```#seek```, etc)
164
+ - Renamed ```after_fetched``` to ```after_fetch``` to normalize the naming convention
165
+ - Instrumentation on a connection delegator level
166
+ - Added ```params_batch#last``` method to retrieve last element after unparsing
167
+ - All params keys are now strings
168
+
169
+ ## 1.1.2
170
+ - #256 - Default kafka.seed_brokers configuration is created in invalid format
171
+
172
+ ## 1.1.1
173
+ - #253 - Allow providing a global per app parser in config settings
174
+
175
+ ## 1.1.0
176
+ - Gem bump
177
+ - Switch from Celluloid to native Thread management
178
+ - Improved shutdown process
179
+ - Introduced optional fetch callbacks and moved current the ```after_received``` there as well
180
+ - Karafka will raise Errors::InvalidPauseTimeout exception when trying to pause but timeout set to 0
181
+ - Allow float for timeouts and other time based second settings
182
+ - Renamed MessagesProcessor to Processor and MessagesConsumer to Consumer - we don't process and don't consumer anything else so it was pointless to keep this "namespace"
183
+ - #232 - Remove unused ActiveSupport require
184
+ - #214 - Expose consumer on a controller layer
185
+ - #193 - Process shutdown callbacks
186
+ - Fixed accessibility of ```#params_batch``` from the outside of the controller
187
+ - connection_pool config options are no longer required
188
+ - celluloid config options are no longer required
189
+ - ```#perform``` is now renamed to ```#consume``` with warning level on using the old one (deprecated)
190
+ - #235 - Rename perform to consume
191
+ - Upgrade to ruby-kafka 0.5
192
+ - Due to redesign of Waterdrop concurrency setting is no longer needed
193
+ - #236 - Manual offset management
194
+ - WaterDrop 1.0.0 support with async
195
+ - Renamed ```batch_consuming``` option to ```batch_fetching``` as it is not a consumption (with processing) but a process of fetching messages from Kafka. The messages is considered consumed, when it is processed.
196
+ - Renamed ```batch_processing``` to ```batch_consuming``` to resemble Kafka concept of consuming messages.
197
+ - Renamed ```after_received``` to ```after_fetched``` to normalize the naming conventions.
198
+ - Responders support the per topic ```async``` option.
199
+
200
+ ## 1.0.1
201
+ - #210 - LoadError: cannot load such file -- [...]/karafka.rb
202
+ - Ruby 2.4.2 as a default (+travis integration)
203
+ - JRuby upgrade
204
+ - Expanded persistence layer (moved to a namespace for easier future development)
205
+ - #213 - Misleading error when non-existing dependency is required
206
+ - #212 - Make params react to #topic, #partition, #offset
207
+ - #215 - Consumer group route dynamic options are ignored
208
+ - #217 - check RUBY_ENGINE constant if RUBY_VERSION is missing (#217)
209
+ - #218 - add configuration setting to control Celluloid's shutdown timeout
210
+ - Renamed Karafka::Routing::Mapper to Karafka::Routing::TopicMapper to match naming conventions
211
+ - #219 - Allow explicit consumer group names, without prefixes
212
+ - Fix to early removed pid upon shutdown of demonized process
213
+ - max_wait_time updated to match https://github.com/zendesk/ruby-kafka/issues/433
214
+ - #230 - Better uri validation for seed brokers (incompatibility as the kafka:// or kafka+ssl:// is required)
215
+ - Small internal docs fixes
216
+ - Dry::Validation::MissingMessageError: message for broker_schema? was not found
217
+ - #238 - warning: already initialized constant Karafka::Schemas::URI_SCHEMES
218
+
219
+ ## 1.0.0
220
+
221
+ ### Closed issues:
222
+
223
+ - #103 - Env for logger is loaded 2 early (on gem load not on app init)
224
+ - #142 - Possibility to better control Kafka consumers (consumer groups management)
225
+ - #150 - Add support for start_from_beginning on a per topic basis
226
+ - #154 - Support for min_bytes and max_wait_time on messages consuming
227
+ - #160 - Reorganize settings to better resemble ruby-kafka requirements
228
+ - #164 - If we decide to have configuration per topic, topic uniqueness should be removed
229
+ - #165 - Router validator
230
+ - #166 - Params and route reorganization (new API)
231
+ - #167 - Remove Sidekiq UI from Karafka
232
+ - #168 - Introduce unique IDs of routes
233
+ - #171 - Add kafka message metadata to params
234
+ - #176 - Transform Karafka::Connection::Consumer into a module
235
+ - #177 - Monitor not reacting when kafka killed with -9
236
+ - #175 - Allow single consumer to subscribe to multiple topics
237
+ - #178 - Remove parsing failover when cannot unparse data
238
+ - #174 - Extended config validation
239
+ - ~~#180 - Switch from JSON parser to yajl-ruby~~
240
+ - #181 - When responder is defined and not used due to ```respond_with``` not being triggered in the perform, it won't raise an exception.
241
+ - #188 - Rename name in config to client id
242
+ - #186 - Support ruby-kafka ```ssl_ca_cert_file_path``` config
243
+ - #189 - karafka console does not preserve history on exit
244
+ - #191 - Karafka 0.6.0rc1 does not work with jruby / now it does :-)
245
+ - Switch to multi json so everyone can use their favourite JSON parser
246
+ - Added jruby support in general and in Travis
247
+ - #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
248
+ - #96 - Karafka server - possibility to run it only for a certain topics
249
+ - ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
250
+ - (optional) pausing upon processing failures ```pause_timeout```
251
+ - Karafka console main process no longer intercepts irb errors
252
+ - Wiki updates
253
+ - #204 - Long running controllers
254
+ - Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
255
+ - #207 - Rename before_enqueued to after_received
256
+ - #147 - De-attach Karafka from Sidekiq by extracting Sidekiq backend
257
+
258
+ ### New features and improvements
259
+
260
+ - batch processing thanks to ```#batch_consuming``` flag and ```#params_batch``` on controllers
261
+ - ```#topic``` method on an controller instance to make a clear distinction in between params and route details
262
+ - Changed routing model (still compatible with 0.5) to allow better resources management
263
+ - Lower memory requirements due to object creation limitation (2-3 times less objects on each new message)
264
+ - Introduced the ```#batch_consuming``` config flag (config for #126) that can be set per each consumer_group
265
+ - Added support for partition, offset and partition key in the params hash
266
+ - ```name``` option in config renamed to ```client_id```
267
+ - Long running controllers with ```persistent``` flag on a topic config level, to make controller instances persistent between messages batches (single controller instance per topic per partition no per messages batch) - turned on by default
268
+
269
+ ### Incompatibilities
270
+
271
+ - Default boot file is renamed from app.rb to karafka.rb
272
+ - Removed worker glass as dependency (now and independent gem)
273
+ - ```kafka.hosts``` option renamed to ```kafka.seed_brokers``` - you don't need to provide all the hosts to work with Kafka
274
+ - ```start_from_beginning``` moved into kafka scope (```kafka.start_from_beginning```)
275
+ - Router no longer checks for route uniqueness - now you can define same routes for multiple kafkas and do a lot of crazy stuff, so it's your responsibility to check uniqueness
276
+ - Change in the way we identify topics in between Karafka and Sidekiq workers. If you upgrade, please make sure, all the jobs scheduled in Sidekiq are finished before the upgrade.
277
+ - ```batch_mode``` renamed to ```batch_fetching```
278
+ - Renamed content to value to better resemble ruby-kafka internal messages naming convention
279
+ - When having a responder with ```required``` topics and not using ```#respond_with``` at all, it will raise an exception
280
+ - Renamed ```inline_mode``` to ```inline_processing``` to resemble other settings conventions
281
+ - Renamed ```inline_processing``` to ```backend``` to reach 1.0 future compatibility
282
+ - Single controller **needs** to be used for a single topic consumption
283
+ - Renamed ```before_enqueue``` to ```after_received``` to better resemble internal logic, since for inline backend, there is no enqueue.
284
+ - Due to the level on which topic and controller are related (class level), the dynamic worker selection is no longer available.
285
+ - Renamed params #retrieve to params #retrieve! to better reflect what it does
286
+
287
+ ### Other changes
288
+ - PolishGeeksDevTools removed (in favour of Coditsu)
289
+ - Waaaaaay better code quality thanks to switching from dev tools to Coditsu
290
+ - Gem bump
291
+ - Cleaner internal API
292
+ - SRP
293
+ - Better settings proxying and management between ruby-kafka and karafka
294
+ - All internal validations are now powered by dry-validation
295
+ - Better naming conventions to reflect Kafka reality
296
+ - Removed Karafka::Connection::Message in favour of direct message details extraction from Kafka::FetchedMessage
297
+
298
+ ## 0.5.0.3
299
+ - #132 - When Kafka is gone, should reconnect after a time period
300
+ - #136 - new ruby-kafka version + other gem bumps
301
+ - ruby-kafka update
302
+ - #135 - NonMatchingRouteError - better error description in the code
303
+ - #140 - Move Capistrano Karafka to a different specific gem
304
+ - #110 - Add call method on a responder class to alias instance build and call
305
+ - #76 - Configs validator
306
+ - #138 - Possibility to have no worker class defined if inline_mode is being used
307
+ - #145 - Topic Mapper
308
+ - Ruby update to 2.4.1
309
+ - Gem bump x2
310
+ - #158 - Update docs section on heroku usage
311
+ - #150 - Add support for start_from_beginning on a per topic basis
312
+ - #148 - Lower Karafka Sidekiq dependency
313
+ - Allow karafka root to be specified from ENV
314
+ - Handle SIGTERM as a shutdown command for kafka server to support Heroku deployment
315
+
316
+ ## 0.5.0.2
317
+ - Gems update x3
318
+ - Default Ruby set to 2.3.3
319
+ - ~~Default Ruby set to 2.4.0~~
320
+ - Readme updates to match bug fixes and resolved issues
321
+ - #95 - Allow options into responder
322
+ - #98 - Use parser when responding on a topic
323
+ - #114 - Option to configure waterdrop connection pool timeout and concurrency
324
+ - #118 - Added dot in topic validation format
325
+ - #119 - add support for authentication using SSL
326
+ - #121 - JSON as a default for standalone responders usage
327
+ - #122 - Allow on capistrano role customization
328
+ - #125 - Add support to batch incoming messages
329
+ - #130 - start_from_beginning flag on routes and default
330
+ - #128 - Monitor caller_label not working with super on inheritance
331
+ - Renamed *inline* to *inline_mode* to stay consistent with flags that change the way karafka works (#125)
332
+ - Dry-configurable bump to 0.5 with fixed proc value evaluation on retrieve patch (internal change)
333
+
334
+ ## 0.5.0.1
335
+ - Fixed inconsistency in responders non-required topic definition. Now only required: false available
336
+ - #101 - Responders fail when multiple_usage true and required false
337
+ - fix error on startup from waterdrop #102
338
+ - Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
339
+ - #105 - Karafka::Monitor#caller_label not working with inherited monitors
340
+ - #99 - Standalone mode (without Sidekiq)
341
+ - #97 - Buffer responders single topics before send (pre-validation)
342
+ - Better control over consumer thanks to additional config options
343
+ - #111 - Dynamic worker assignment based on the income params
344
+ - Long shutdown time fix
345
+
346
+ ## 0.5.0
347
+ - Removed Zookeeper totally as dependency
348
+ - Better group and partition rebalancing
349
+ - Automatic thread management (no need for tuning) - each topic is a separate actor/thread
350
+ - Moved from Poseidon into Ruby-Kafka
351
+ - No more max_concurrency setting
352
+ - After you define your App class and routes (and everything else) you need to add execute App.boot!
353
+ - Manual consuming is no longer available (no more karafka consume command)
354
+ - Karafka topics CLI is no longer available. No Zookeeper - no global topic discovery
355
+ - Dropped ZK as dependency
356
+ - karafka info command no longer prints details about Zookeeper
357
+ - Better shutdown
358
+ - No more autodiscovery via Zookeeper - instead, the whole cluster will be discovered directly from Kafka
359
+ - No more support for Kafka 0.8
360
+ - Support for Kafka 0.9
361
+ - No more need for ActorCluster, since now we have a single thread (and Kafka connection) per topic
362
+ - Ruby 2.2.* support dropped
363
+ - Using App name as a Kafka client_id
364
+ - Automatic Capistrano integration
365
+ - Responders support for handling better responses pipe-lining and better responses flow description and design (see README for more details)
366
+ - Gem bump
367
+ - Readme updates
368
+ - karafka flow CLI command for printing the application flow
369
+ - Some internal refactoring
370
+
371
+ ## 0.4.2
372
+ - #87 - Re-consume mode with crone for better Rails/Rack integration
373
+ - Moved Karafka server related stuff into separate Karafka::Server class
374
+ - Renamed Karafka::Runner into Karafka::Fetcher
375
+ - Gem bump
376
+ - Added chroot option to Zookeeper options
377
+ - Moved BROKERS_PATH into config from constant
378
+ - Added Karafka consume CLI action for a short running single consumption round
379
+ - Small fixes to close broken connections
380
+ - Readme updates
381
+
382
+ ## 0.4.1
383
+ - Explicit throw(:abort) required to halt before_enqueue (like in Rails 5)
384
+ - #61 - autodiscovery of Kafka brokers based on Zookeeper data
385
+ - #63 - Graceful shutdown with current offset state during data processing
386
+ - #65 - Example of NewRelic monitor is outdated
387
+ - #71 - Setup should be executed after user code is loaded
388
+ - Gem bump x3
389
+ - Rubocop remarks
390
+ - worker_timeout config option has been removed. It now needs to be defined manually by the framework user because WorkerGlass::Timeout can be disabled and we cannot use Karafka settings on a class level to initialize user code stuff
391
+ - Moved setup logic under setup/Setup namespace
392
+ - Better defaults handling
393
+ - #75 - Kafka and Zookeeper options as a hash
394
+ - #82 - Karafka autodiscovery fails upon caching of configs
395
+ - #81 - Switch config management to dry configurable
396
+ - Version fix
397
+ - Dropped support for Ruby 2.1.*
398
+ - Ruby bump to 2.3.1
399
+
400
+ ## 0.4.0
401
+ - Added WaterDrop gem with default configuration
402
+ - Refactoring of config logic to simplify adding new dependencies that need to be configured based on #setup data
403
+ - Gem bump
404
+ - Readme updates
405
+ - Renamed cluster to actor_cluster for method names
406
+ - Replaced SidekiqGlass with generic WorkerGlass lib
407
+ - Application bootstrap in app.rb no longer required
408
+ - Karafka.boot needs to be executed after all the application files are loaded (template updated)
409
+ - Small loader refactor (no API changes)
410
+ - Ruby 2.3.0 support (default)
411
+ - No more rake tasks
412
+ - Karafka CLI instead of rake tasks
413
+ - Worker cli command allows passing additional options directly to Sidekiq
414
+ - Renamed concurrency to max_concurrency - it describes better what happens - Karafka will use this number of threads only when required
415
+ - Added wait_timeout that allows us to tune how long should we wait on a single socket connection (single topic) for new messages before going to next one (this applies to each thread separately)
416
+ - Rubocop remarks
417
+ - Removed Sinatra and Puma dependencies
418
+ - Karafka Cli internal reorganization
419
+ - Karafka Cli routes task
420
+ - #37 - warn log for failed parsing of a message
421
+ - #43 - wrong constant name
422
+ - #44 - Method name conflict
423
+ - #48 - Cannot load such file -- celluloid/current
424
+ - #46 - Loading application
425
+ - #45 - Set up monitor in config
426
+ - #47 - rake karafka:run uses app.rb only
427
+ - #53 - README update with Sinatra/Rails integration description
428
+ - #41 - New Routing engine
429
+ - #54 - Move Karafka::Workers::BaseWorker to Karafka::BaseWorker
430
+ - #55 - ApplicationController and ApplicationWorker
431
+
432
+ ## 0.3.2
433
+ - Karafka::Params::Params lazy load merge keys with string/symbol names priorities fix
434
+
435
+ ## 0.3.1
436
+ - Renamed Karafka::Monitor to Karafka::Process to represent a Karafka process wrapper
437
+ - Added Karafka::Monitoring that allows to add custom logging and monitoring with external libraries and systems
438
+ - Moved logging functionality into Karafka::Monitoring default monitoring
439
+ - Added possibility to provide own monitoring as long as in responds to #notice and #notice_error
440
+ - Standardized logging format for all logs
441
+
442
+ ## 0.3.0
443
+ - Switched from custom ParserError for each parser to general catching of Karafka::Errors::ParseError and its descendants
444
+ - Gem bump
445
+ - Fixed #32 - now when using custom workers that does not inherit from Karafka::BaseWorker perform method is not required. Using custom workers means that the logic that would normally lie under #perform, needs to be executed directly from the worker.
446
+ - Fixed #31 - Technically didn't fix because this is how Sidekiq is meant to work, but provided possibility to assign custom interchangers that allow to bypass JSON encoding issues by converting data that goes to Redis to a required format (and parsing it back when it is fetched)
447
+ - Added full parameters lazy load - content is no longer loaded during #perform_async if params are not used in before_enqueue
448
+ - No more namespaces for Redis by default (use separate DBs)
449
+
450
+ ## 0.1.21
451
+ - Sidekiq 4.0.1 bump
452
+ - Gem bump
453
+ - Added direct celluloid requirement to Karafka (removed from Sidekiq)
454
+
455
+ ## 0.1.19
456
+ - Internal call - schedule naming change
457
+ - Enqueue to perform_async naming in controller to follow Sidekiq naming convention
458
+ - Gem bump
459
+
460
+ ## 0.1.18
461
+ - Changed Redis configuration options into a single hash that is directly passed to Redis setup for Sidekiq
462
+ - Added config.ru to provide a Sidekiq web UI (see README for more details)
463
+
464
+ ## 0.1.17
465
+ - Changed Karafka::Connection::Cluster tp Karafka::Connection::ActorCluster to distinguish between a single thread actor cluster for multiple topic connection and a future feature that will allow process clusterization.
466
+ - Add an ability to use user-defined parsers for a messages
467
+ - Lazy load params for before callbacks
468
+ - Automatic loading/initializing all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
469
+ - Params are now private to controller
470
+ - Added bootstrap method to app.rb
471
+
472
+ ## 0.1.16
473
+ - Cluster level error catching for all exceptions so actor is not killer
474
+ - Cluster level error logging
475
+ - Listener refactoring (QueueConsumer extracted)
476
+ - Karafka::Connection::QueueConsumer to wrap around fetching logic - technically we could replace Kafka with any other messaging engine as long as we preserve the same API
477
+ - Added debug env for debugging purpose in applications
478
+
479
+ ## 0.1.15
480
+ - Fixed max_wait_ms vs socket_timeout_ms issue
481
+ - Fixed closing queue connection after Poseidon::Errors::ProtocolError failure
482
+ - Fixed wrong logging file selection based on env
483
+ - Extracted Karafka::Connection::QueueConsumer object to wrap around queue connection
484
+
485
+ ## 0.1.14
486
+ - Rake tasks for listing all the topics on Kafka server (rake kafka:topics)
487
+
488
+ ## 0.1.13
489
+ - Ability to assign custom workers and use them bypassing Karafka::BaseWorker (or its descendants)
490
+ - Gem bump
491
+
492
+ ## 0.1.12
493
+ - All internal errors went to Karafka::Errors namespace
494
+
495
+ ## 0.1.11
496
+ - Rescuing all the "before Sidekiq" processing so errors won't affect other incoming messages
497
+ - Fixed dying actors after connection error
498
+ - Added a new app status - "initializing"
499
+ - Karafka::Status model cleanup
500
+
501
+ ## 0.1.10
502
+ - Added possibility to specify redis namespace in configuration (failover to app name)
503
+ - Renamed redis_host to redis_url in configuration
504
+
505
+ ## 0.1.9
506
+ - Added worker logger
507
+
508
+ ## 0.1.8
509
+ - Dropped local env support in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
510
+
511
+ ## 0.1.7
512
+ - Karafka option for Redis hosts (not localhost only)
513
+
514
+ ## 0.1.6
515
+ - Added better concurency by clusterization of listeners
516
+ - Added graceful shutdown
517
+ - Added concurency that allows to handle bigger applications with celluloid
518
+ - Karafka controllers no longer require group to be defined (created based on the topic and app name)
519
+ - Karafka controllers no longer require topic to be defined (created based on the controller name)
520
+ - Readme updates
521
+
522
+ ## 0.1.5
523
+ - Celluloid support for listeners
524
+ - Multi target logging (STDOUT and file)
525
+
526
+ ## 0.1.4
527
+ - Renamed events to messages to follow Apache Kafka naming convention
528
+
529
+ ## 0.1.3
530
+ - Karafka::App.logger moved to Karafka.logger
531
+ - README updates (Usage section was added)
532
+
533
+ ## 0.1.2
534
+ - Logging to log/environment.log
535
+ - Karafka::Runner
536
+
537
+ ## 0.1.1
538
+ - README updates
539
+ - Rake tasks updates
540
+ - Rake installation task
541
+ - Changelog file added
542
+
543
+ ## 0.1.0
544
+ - Initial framework code