karafka 1.3.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +2 -0
  3. data.tar.gz.sig +0 -0
  4. data/.coditsu/ci.yml +3 -0
  5. data/.console_irbrc +11 -0
  6. data/.github/FUNDING.yml +3 -0
  7. data/.github/ISSUE_TEMPLATE/bug_report.md +50 -0
  8. data/.github/ISSUE_TEMPLATE/feature_request.md +20 -0
  9. data/.gitignore +69 -0
  10. data/.rspec +1 -0
  11. data/.ruby-gemset +1 -0
  12. data/.ruby-version +1 -0
  13. data/.travis.yml +36 -0
  14. data/CHANGELOG.md +540 -0
  15. data/CODE_OF_CONDUCT.md +46 -0
  16. data/CONTRIBUTING.md +41 -0
  17. data/Gemfile +12 -0
  18. data/Gemfile.lock +140 -0
  19. data/MIT-LICENCE +18 -0
  20. data/README.md +101 -0
  21. data/bin/karafka +19 -0
  22. data/certs/mensfeld.pem +25 -0
  23. data/config/errors.yml +39 -0
  24. data/karafka.gemspec +44 -0
  25. data/lib/karafka.rb +72 -0
  26. data/lib/karafka/app.rb +53 -0
  27. data/lib/karafka/attributes_map.rb +62 -0
  28. data/lib/karafka/backends/inline.rb +16 -0
  29. data/lib/karafka/base_consumer.rb +57 -0
  30. data/lib/karafka/base_responder.rb +226 -0
  31. data/lib/karafka/cli.rb +54 -0
  32. data/lib/karafka/cli/base.rb +78 -0
  33. data/lib/karafka/cli/console.rb +31 -0
  34. data/lib/karafka/cli/flow.rb +49 -0
  35. data/lib/karafka/cli/info.rb +31 -0
  36. data/lib/karafka/cli/install.rb +66 -0
  37. data/lib/karafka/cli/server.rb +71 -0
  38. data/lib/karafka/code_reloader.rb +67 -0
  39. data/lib/karafka/connection/api_adapter.rb +155 -0
  40. data/lib/karafka/connection/batch_delegator.rb +51 -0
  41. data/lib/karafka/connection/builder.rb +16 -0
  42. data/lib/karafka/connection/client.rb +117 -0
  43. data/lib/karafka/connection/listener.rb +71 -0
  44. data/lib/karafka/connection/message_delegator.rb +36 -0
  45. data/lib/karafka/consumers/callbacks.rb +71 -0
  46. data/lib/karafka/consumers/includer.rb +63 -0
  47. data/lib/karafka/consumers/metadata.rb +10 -0
  48. data/lib/karafka/consumers/responders.rb +24 -0
  49. data/lib/karafka/consumers/single_params.rb +15 -0
  50. data/lib/karafka/contracts.rb +10 -0
  51. data/lib/karafka/contracts/config.rb +21 -0
  52. data/lib/karafka/contracts/consumer_group.rb +206 -0
  53. data/lib/karafka/contracts/consumer_group_topic.rb +19 -0
  54. data/lib/karafka/contracts/responder_usage.rb +54 -0
  55. data/lib/karafka/contracts/server_cli_options.rb +31 -0
  56. data/lib/karafka/errors.rb +51 -0
  57. data/lib/karafka/fetcher.rb +42 -0
  58. data/lib/karafka/helpers/class_matcher.rb +88 -0
  59. data/lib/karafka/helpers/config_retriever.rb +46 -0
  60. data/lib/karafka/helpers/inflector.rb +26 -0
  61. data/lib/karafka/helpers/multi_delegator.rb +32 -0
  62. data/lib/karafka/instrumentation/logger.rb +58 -0
  63. data/lib/karafka/instrumentation/monitor.rb +70 -0
  64. data/lib/karafka/instrumentation/proctitle_listener.rb +36 -0
  65. data/lib/karafka/instrumentation/stdout_listener.rb +138 -0
  66. data/lib/karafka/params/builders/metadata.rb +33 -0
  67. data/lib/karafka/params/builders/params.rb +36 -0
  68. data/lib/karafka/params/builders/params_batch.rb +25 -0
  69. data/lib/karafka/params/metadata.rb +35 -0
  70. data/lib/karafka/params/params.rb +68 -0
  71. data/lib/karafka/params/params_batch.rb +61 -0
  72. data/lib/karafka/patches/ruby_kafka.rb +47 -0
  73. data/lib/karafka/persistence/client.rb +29 -0
  74. data/lib/karafka/persistence/consumers.rb +45 -0
  75. data/lib/karafka/persistence/topics.rb +48 -0
  76. data/lib/karafka/process.rb +60 -0
  77. data/lib/karafka/responders/builder.rb +36 -0
  78. data/lib/karafka/responders/topic.rb +55 -0
  79. data/lib/karafka/routing/builder.rb +89 -0
  80. data/lib/karafka/routing/consumer_group.rb +61 -0
  81. data/lib/karafka/routing/consumer_mapper.rb +34 -0
  82. data/lib/karafka/routing/proxy.rb +46 -0
  83. data/lib/karafka/routing/router.rb +29 -0
  84. data/lib/karafka/routing/topic.rb +62 -0
  85. data/lib/karafka/routing/topic_mapper.rb +53 -0
  86. data/lib/karafka/serialization/json/deserializer.rb +27 -0
  87. data/lib/karafka/serialization/json/serializer.rb +31 -0
  88. data/lib/karafka/server.rb +86 -0
  89. data/lib/karafka/setup/config.rb +221 -0
  90. data/lib/karafka/setup/configurators/water_drop.rb +36 -0
  91. data/lib/karafka/setup/dsl.rb +21 -0
  92. data/lib/karafka/status.rb +29 -0
  93. data/lib/karafka/templates/application_consumer.rb.erb +7 -0
  94. data/lib/karafka/templates/application_responder.rb.erb +11 -0
  95. data/lib/karafka/templates/karafka.rb.erb +92 -0
  96. data/lib/karafka/version.rb +7 -0
  97. data/log/.gitkeep +0 -0
  98. metadata +336 -0
  99. metadata.gz.sig +2 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: dc0f99f61db358d26a8e65c677c387a4dc18ada9fb3a0fba91d512408fe6045c
4
+ data.tar.gz: 9c5f338634cf431aa8f69b75cc18540a6a223cfb613691a92c4ce4eacb3a8914
5
+ SHA512:
6
+ metadata.gz: 41f1615ac9cf15f9a8fe8b511836399634298a94c6249e9ac09449d82138c5af073e593b127129fa9bf7ed336758b2095372c0fb271a998d7db843e6f8a53a2c
7
+ data.tar.gz: cbf4dbfb337cda933e21d9ecd6ffd3f37fbe58b4ced9a0b48a5124837de2b8656ca0caa0a1f94fc5c14c4d6e9c2073e25c94c224571d4a514bab9bb052fed9f2
@@ -0,0 +1,2 @@
1
+ C� be�n�4�+�b|��Q-�[fQy[�L��^8�'��&�ZǴ˩ ���w�t8�������сb�q�d=U�h�gڻ��x�λ 7�l_�����5g~�^>�3��N;�#/����P�2a�xۍ�5ۿ�bn��������$�wvr�~_e�֍�|U/��]p(���d �m&���O$�3�+�ٷ�{�hlF��#QsW"��N��������sFRg�i3���)��g�
2
+ �C���v�ƈ���-�^e�R���9�$��|Ǫ�� ����⊈����48�=�M� �X �mkݪ��])�]*���@�=�{�%�J�(��g�c���z�޹U@W�+�U�dImM�SZ�˫>�T!�����+��09Ė�tR���`q�!
Binary file
@@ -0,0 +1,3 @@
1
+ repository_id: 'd4482d42-f6b5-44ba-a5e4-00989ac519ee'
2
+ api_key: <%= ENV['CODITSU_API_KEY'] %>
3
+ api_secret: <%= ENV['CODITSU_API_SECRET'] %>
@@ -0,0 +1,11 @@
1
+ # irbrc for Karafka console
2
+
3
+ IRB.conf[:AUTO_INDENT] = true
4
+ IRB.conf[:SAVE_HISTORY] = 1000
5
+ IRB.conf[:USE_READLINE] = true
6
+ IRB.conf[:HISTORY_FILE] = ".irb-history"
7
+ IRB.conf[:LOAD_MODULES] = [] unless IRB.conf.key?(:LOAD_MODULES)
8
+
9
+ unless IRB.conf[:LOAD_MODULES].include?('irb/completion')
10
+ IRB.conf[:LOAD_MODULES] << 'irb/completion'
11
+ end
@@ -0,0 +1,3 @@
1
+ # These are supported funding model platforms
2
+
3
+ open_collective: karafka
@@ -0,0 +1,50 @@
1
+ ---
2
+ name: Bug Report
3
+ about: Report an issue with Karafka you've discovered.
4
+ ---
5
+
6
+ *Be clear, concise and precise in your description of the problem.
7
+ Open an issue with a descriptive title and a summary in grammatically correct,
8
+ complete sentences.*
9
+
10
+ *Use the template below when reporting bugs. Please, make sure that
11
+ you're running the latest stable Karafka and that the problem you're reporting
12
+ hasn't been reported (and potentially fixed) already.*
13
+
14
+ *Before filing the ticket you should replace all text above the horizontal
15
+ rule with your own words.*
16
+
17
+ --------
18
+
19
+ ## Expected behavior
20
+
21
+ Describe here how you expected Karafka to behave in this particular situation.
22
+
23
+ ## Actual behavior
24
+
25
+ Describe here what actually happened.
26
+
27
+ ## Steps to reproduce the problem
28
+
29
+ This is extremely important! Providing us with a reliable way to reproduce
30
+ a problem will expedite its solution.
31
+
32
+ ## Your setup details
33
+
34
+ Please provide kafka version and the output of `karafka info` or `bundle exec karafka info` if using Bundler.
35
+
36
+ Here's an example:
37
+
38
+ ```
39
+ $ [bundle exec] karafka info
40
+ Karafka version: 1.3.0
41
+ Ruby version: 2.6.3
42
+ Ruby-kafka version: 0.7.9
43
+ Application client id: karafka-local
44
+ Backend: inline
45
+ Batch fetching: true
46
+ Batch consuming: true
47
+ Boot file: /app/karafka/karafka.rb
48
+ Environment: development
49
+ Kafka seed brokers: ["kafka://kafka:9092"]
50
+ ```
@@ -0,0 +1,20 @@
1
+ ---
2
+ name: Feature Request
3
+ about: Suggest new Karafka features or improvements to existing features.
4
+ ---
5
+
6
+ ## Is your feature request related to a problem? Please describe.
7
+
8
+ A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
9
+
10
+ ## Describe the solution you'd like
11
+
12
+ A clear and concise description of what you want to happen.
13
+
14
+ ## Describe alternatives you've considered
15
+
16
+ A clear and concise description of any alternative solutions or features you've considered.
17
+
18
+ ## Additional context
19
+
20
+ Add any other context or screenshots about the feature request here.
@@ -0,0 +1,69 @@
1
+ # bundler state
2
+ /.bundle
3
+ /vendor/bundle/
4
+ /vendor/ruby/
5
+ /ruby/
6
+ app.god
7
+
8
+ # minimal Rails specific artifacts
9
+ /.coditsu/local.yml
10
+ db/*.sqlite3
11
+ /log/development.log
12
+ /log/production.log
13
+ /log/test.log
14
+ /tmp/*
15
+ *.gem
16
+ *.~
17
+
18
+ # various artifacts
19
+ **.war
20
+ *.rbc
21
+ *.sassc
22
+ .byebug_history
23
+ .redcar/
24
+ .capistrano/
25
+ .sass-cache
26
+ /config/god/sidekiq.rb
27
+ /config/puma.rb
28
+ /coverage.data
29
+ /coverage/
30
+ /doc/api/
31
+ /doc/app/
32
+ /doc/yard
33
+ /doc/features.html
34
+ /doc/specs.html
35
+ /spec/tmp/*
36
+ /cache
37
+ /capybara*
38
+ /capybara-*.html
39
+ /gems
40
+ /specifications
41
+ rerun.txt
42
+ pickle-email-*.html
43
+
44
+ # If you find yourself ignoring temporary files generated by your text editor
45
+ # or operating system, you probably want to add a global ignore instead:
46
+ # git config --global core.excludesfile ~/.gitignore_global
47
+ #
48
+ # Here are some files you may want to ignore globally:
49
+
50
+ # scm revert files
51
+ **.orig
52
+
53
+ # Mac finder artifacts
54
+ .DS_Store
55
+
56
+ # Netbeans project directory
57
+ /nbproject
58
+
59
+ # RubyMine project files
60
+ .idea
61
+
62
+ # Textmate project files
63
+ /*.tmproj
64
+
65
+ # vim artifacts
66
+ **.swp
67
+
68
+ # documentation
69
+ .yardoc
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --require spec_helper
@@ -0,0 +1 @@
1
+ karafka
@@ -0,0 +1 @@
1
+ 2.7.1
@@ -0,0 +1,36 @@
1
+ services:
2
+ - docker
3
+
4
+ dist: trusty
5
+ sudo: false
6
+ cache: bundler
7
+
8
+ git:
9
+ depth: false
10
+
11
+ test: &test
12
+ stage: Test
13
+ language: ruby
14
+ before_install:
15
+ - yes | gem update --system
16
+ script: bundle exec rspec
17
+
18
+ jobs:
19
+ include:
20
+ - <<: *test
21
+ rvm: 2.7.1
22
+ - <<: *test
23
+ rvm: 2.6.6
24
+ - <<: *test
25
+ rvm: 2.5.8
26
+
27
+ - stage: coditsu
28
+ language: ruby
29
+ rvm: 2.7.1
30
+ before_install:
31
+ - yes | gem update --system
32
+ script: \curl -sSL https://api.coditsu.io/run/ci | bash
33
+
34
+ stages:
35
+ - test
36
+ - coditsu
@@ -0,0 +1,540 @@
1
+ # Karafka framework changelog
2
+
3
+ ## 1.3.5 (2020-04-02)
4
+ - #578 - ThreadError: can't be called from trap context patch
5
+
6
+ ## 1.3.4 (2020-02-17)
7
+ - `dry-configurable` upgrade (solnic)
8
+ - Remove temporary `thor` patches that are no longer needed
9
+
10
+ ## 1.3.3 (2019-12-23)
11
+ - Require `delegate` to fix missing dependency in `ruby-kafka`
12
+
13
+ ## 1.3.2 (2019-12-23)
14
+ - #561 - Allow `thor` 1.0.x usage in Karafka
15
+ - #567 - Ruby 2.7.0 support + unfreeze of a frozen string fix
16
+
17
+ ## 1.3.1 (2019-11-11)
18
+ - #545 - Makes sure the log directory exists when is possible (robertomiranda)
19
+ - Ruby 2.6.5 support
20
+ - #551 - add support for DSA keys
21
+ - #549 - Missing directories after `karafka install` (nijikon)
22
+
23
+ ## 1.3.0 (2019-09-09)
24
+ - Drop support for Ruby 2.4
25
+ - YARD docs tags cleanup
26
+
27
+ ## 1.3.0.rc1 (2019-07-31)
28
+ - Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
29
+ - Update ruby-kafka to the 0.7 version
30
+ - Support messages headers receiving
31
+ - Message bus unification
32
+ - Parser available in metadata
33
+ - Cleanup towards moving to a non-global state app management
34
+ - Drop Ruby 2.3 support
35
+ - Support for Ruby 2.6.3
36
+ - `Karafka::Loader` has been removed in favor of Zeitwerk
37
+ - Schemas are now contracts
38
+ - #393 - Reorganize responders - removed `multiple_usage` constrain
39
+ - #388 - ssl_client_cert_chain sync
40
+ - #300 - Store value in a value key and replace its content with parsed version - without root merge
41
+ - #331 - Disallow building groups without topics
42
+ - #340 - Instrumentation unification. Better and more consistent naming
43
+ - #340 - Procline instrumentation for a nicer process name
44
+ - #342 - Change default for `fetcher_max_queue_size` from `100` to `10` to lower max memory usage
45
+ - #345 - Cleanup exceptions names
46
+ - #341 - Split connection delegator into batch delegator and single_delegator
47
+ - #351 - Rename `#retrieve!` to `#parse!` on params and `#parsed` to `parse!` on params batch.
48
+ - #351 - Adds '#first' for params_batch that returns parsed first element from the params_batch object.
49
+ - #360 - Single params consuming mode automatically parses data specs
50
+ - #359 - Divide mark_as_consumed into mark_as_consumed and mark_as_consumed!
51
+ - #356 - Provide a `#values` for params_batch to extract only values of objects from the params_batch
52
+ - #363 - Too shallow ruby-kafka version lock
53
+ - #354 - Expose consumer heartbeat
54
+ - #377 - Remove the persistent setup in favor of persistence
55
+ - #375 - Sidekiq Backend parser mismatch
56
+ - #369 - Single consumer can support more than one topic
57
+ - #288 - Drop dependency on `activesupport` gem
58
+ - #371 - SASL over SSL
59
+ - #392 - Move params redundant data to metadata
60
+ - #335 - Metadata access from within the consumer
61
+ - #402 - Delayed reconnection upon critical failures
62
+ - #405 - `reconnect_timeout` value is now being validated
63
+ - #437 - Specs ensuring that the `#437` won't occur in the `1.3` release
64
+ - #426 - ssl client cert key password
65
+ - #444 - add certificate and private key validation
66
+ - #460 - Decouple responder "parser" (generator?) from topic.parser (benissimo)
67
+ - #463 - Split parsers into serializers / deserializers
68
+ - #473 - Support SASL OAuthBearer Authentication
69
+ - #475 - Disallow subscribing to the same topic with multiple consumers
70
+ - #485 - Setting shutdown_timeout to nil kills the app without waiting for anything
71
+ - #487 - Make listeners as instances
72
+ - #29 - Consumer class names must have the word "Consumer" in it in order to work (Sidekiq backend)
73
+ - #491 - irb is missing for console to work
74
+ - #502 - Karafka process hangs when sending multiple sigkills
75
+ - #506 - ssl_verify_hostname sync
76
+ - #483 - Upgrade dry-validation before releasing 1.3
77
+ - #492 - Use Zeitwerk for code reload in development
78
+ - #508 - Reset the consumers instances upon reconnecting to a cluster
79
+ - [#530](https://github.com/karafka/karafka/pull/530) - expose ruby and ruby-kafka version
80
+ - [534](https://github.com/karafka/karafka/pull/534) - Allow to use headers in the deserializer object
81
+
82
+ ## 1.2.11
83
+ - [#470](https://github.com/karafka/karafka/issues/470) Karafka not working with dry-configurable 0.8
84
+
85
+ ## 1.2.10
86
+ - [#453](https://github.com/karafka/karafka/pull/453) require `Forwardable` module
87
+
88
+ ## 1.2.9
89
+ - Critical exceptions now will cause consumer to stop instead of retrying without a break
90
+ - #412 - Fix dry-inflector dependency lock in gemspec
91
+ - #414 - Backport to 1.2 the delayed retry upon failure
92
+ - #437 - Raw message is no longer added to params after ParserError raised
93
+
94
+ ## 1.2.8
95
+ - #408 - Responder Topic Lookup Bug on Heroku
96
+
97
+ ## 1.2.7
98
+ - Unlock Ruby-kafka version with a warning
99
+
100
+ ## 1.2.6
101
+ - Lock WaterDrop to 1.2.3
102
+ - Lock Ruby-Kafka to 0.6.x (support for 0.7 will be added in Karafka 1.3)
103
+ - #382 - Full logging with AR, etc for development mode when there is Rails integration
104
+
105
+ ## 1.2.5
106
+ - #354 - Expose consumer heartbeat
107
+ - #373 - Async producer not working properly with responders
108
+
109
+ ## 1.2.4
110
+ - #332 - Fetcher for max queue size
111
+
112
+ ## 1.2.3
113
+ - #313 - support PLAINTEXT and SSL for scheme
114
+ - #288 - drop activesupport callbacks in favor of notifications
115
+ - #320 - Pausing indefinitely with nil pause timeout doesn't work
116
+ - #318 - Partition pausing doesn't work with custom topic mappers
117
+ - Rename ConfigAdapter to ApiAdapter to better reflect what it does
118
+ - #317 - Manual offset committing doesn't work with custom topic mappers
119
+ - #319 - Support for exponential backoff in pause
120
+
121
+ ## 1.2.2
122
+ - #312 - Broken for ActiveSupport 5.2.0
123
+
124
+ ## 1.2.1
125
+ - #304 - Unification of error instrumentation event details
126
+ - #306 - Using file logger from within a trap context upon shutdown is impossible
127
+
128
+ ## 1.2.0
129
+ - Spec improvements
130
+ - #260 - Specs missing randomization
131
+ - #251 - Shutdown upon non responding (unreachable) cluster is not possible
132
+ - #258 - Investigate lowering requirements on activesupport
133
+ - #246 - Alias consumer#mark_as_consumed on controller
134
+ - #259 - Allow forcing key/partition key on responders
135
+ - #267 - Styling inconsistency
136
+ - #242 - Support setting the max bytes to fetch per request
137
+ - #247 - Support SCRAM once released
138
+ - #271 - Provide an after_init option to pass a configuration block
139
+ - #262 - Error in the monitor code for NewRelic
140
+ - #241 - Performance metrics
141
+ - #274 - Rename controllers to consumers
142
+ - #184 - Seek to
143
+ - #284 - Dynamic Params parent class
144
+ - #275 - ssl_ca_certs_from_system
145
+ - #296 - Instrument forceful exit with an error
146
+ - Replaced some of the activesupport parts with dry-inflector
147
+ - Lower ActiveSupport dependency
148
+ - Remove configurators in favor of the after_init block configurator
149
+ - Ruby 2.5.0 support
150
+ - Renamed Karafka::Connection::Processor to Karafka::Connection::Delegator to match incoming naming conventions
151
+ - Renamed Karafka::Connection::Consumer to Karafka::Connection::Client due to #274
152
+ - Removed HashWithIndifferentAccess in favor of a regular hash
153
+ - JSON parsing defaults now to string keys
154
+ - Lower memory usage due to less params data internal details
155
+ - Support multiple ```after_init``` blocks in favor of a single one
156
+ - Renamed ```received_at``` to ```receive_time``` to follow ruby-kafka and WaterDrop conventions
157
+ - Adjust internal setup to easier map Ruby-Kafka config changes
158
+ - System callbacks reorganization
159
+ - Added ```before_fetch_loop``` configuration block for early client usage (```#seek```, etc)
160
+ - Renamed ```after_fetched``` to ```after_fetch``` to normalize the naming convention
161
+ - Instrumentation on a connection delegator level
162
+ - Added ```params_batch#last``` method to retrieve last element after unparsing
163
+ - All params keys are now strings
164
+
165
+ ## 1.1.2
166
+ - #256 - Default kafka.seed_brokers configuration is created in invalid format
167
+
168
+ ## 1.1.1
169
+ - #253 - Allow providing a global per app parser in config settings
170
+
171
+ ## 1.1.0
172
+ - Gem bump
173
+ - Switch from Celluloid to native Thread management
174
+ - Improved shutdown process
175
+ - Introduced optional fetch callbacks and moved current the ```after_received``` there as well
176
+ - Karafka will raise Errors::InvalidPauseTimeout exception when trying to pause but timeout set to 0
177
+ - Allow float for timeouts and other time based second settings
178
+ - Renamed MessagesProcessor to Processor and MessagesConsumer to Consumer - we don't process and don't consumer anything else so it was pointless to keep this "namespace"
179
+ - #232 - Remove unused ActiveSupport require
180
+ - #214 - Expose consumer on a controller layer
181
+ - #193 - Process shutdown callbacks
182
+ - Fixed accessibility of ```#params_batch``` from the outside of the controller
183
+ - connection_pool config options are no longer required
184
+ - celluloid config options are no longer required
185
+ - ```#perform``` is now renamed to ```#consume``` with warning level on using the old one (deprecated)
186
+ - #235 - Rename perform to consume
187
+ - Upgrade to ruby-kafka 0.5
188
+ - Due to redesign of Waterdrop concurrency setting is no longer needed
189
+ - #236 - Manual offset management
190
+ - WaterDrop 1.0.0 support with async
191
+ - Renamed ```batch_consuming``` option to ```batch_fetching``` as it is not a consumption (with processing) but a process of fetching messages from Kafka. The messages is considered consumed, when it is processed.
192
+ - Renamed ```batch_processing``` to ```batch_consuming``` to resemble Kafka concept of consuming messages.
193
+ - Renamed ```after_received``` to ```after_fetched``` to normalize the naming conventions.
194
+ - Responders support the per topic ```async``` option.
195
+
196
+ ## 1.0.1
197
+ - #210 - LoadError: cannot load such file -- [...]/karafka.rb
198
+ - Ruby 2.4.2 as a default (+travis integration)
199
+ - JRuby upgrade
200
+ - Expanded persistence layer (moved to a namespace for easier future development)
201
+ - #213 - Misleading error when non-existing dependency is required
202
+ - #212 - Make params react to #topic, #partition, #offset
203
+ - #215 - Consumer group route dynamic options are ignored
204
+ - #217 - check RUBY_ENGINE constant if RUBY_VERSION is missing (#217)
205
+ - #218 - add configuration setting to control Celluloid's shutdown timeout
206
+ - Renamed Karafka::Routing::Mapper to Karafka::Routing::TopicMapper to match naming conventions
207
+ - #219 - Allow explicit consumer group names, without prefixes
208
+ - Fix to early removed pid upon shutdown of demonized process
209
+ - max_wait_time updated to match https://github.com/zendesk/ruby-kafka/issues/433
210
+ - #230 - Better uri validation for seed brokers (incompatibility as the kafka:// or kafka+ssl:// is required)
211
+ - Small internal docs fixes
212
+ - Dry::Validation::MissingMessageError: message for broker_schema? was not found
213
+ - #238 - warning: already initialized constant Karafka::Schemas::URI_SCHEMES
214
+
215
+ ## 1.0.0
216
+
217
+ ### Closed issues:
218
+
219
+ - #103 - Env for logger is loaded 2 early (on gem load not on app init)
220
+ - #142 - Possibility to better control Kafka consumers (consumer groups management)
221
+ - #150 - Add support for start_from_beginning on a per topic basis
222
+ - #154 - Support for min_bytes and max_wait_time on messages consuming
223
+ - #160 - Reorganize settings to better resemble ruby-kafka requirements
224
+ - #164 - If we decide to have configuration per topic, topic uniqueness should be removed
225
+ - #165 - Router validator
226
+ - #166 - Params and route reorganization (new API)
227
+ - #167 - Remove Sidekiq UI from Karafka
228
+ - #168 - Introduce unique IDs of routes
229
+ - #171 - Add kafka message metadata to params
230
+ - #176 - Transform Karafka::Connection::Consumer into a module
231
+ - #177 - Monitor not reacting when kafka killed with -9
232
+ - #175 - Allow single consumer to subscribe to multiple topics
233
+ - #178 - Remove parsing failover when cannot unparse data
234
+ - #174 - Extended config validation
235
+ - ~~#180 - Switch from JSON parser to yajl-ruby~~
236
+ - #181 - When responder is defined and not used due to ```respond_with``` not being triggered in the perform, it won't raise an exception.
237
+ - #188 - Rename name in config to client id
238
+ - #186 - Support ruby-kafka ```ssl_ca_cert_file_path``` config
239
+ - #189 - karafka console does not preserve history on exit
240
+ - #191 - Karafka 0.6.0rc1 does not work with jruby / now it does :-)
241
+ - Switch to multi json so everyone can use their favourite JSON parser
242
+ - Added jruby support in general and in Travis
243
+ - #196 - Topic mapper does not map topics when subscribing thanks to @webandtech
244
+ - #96 - Karafka server - possibility to run it only for a certain topics
245
+ - ~~karafka worker cli option is removed (please use sidekiq directly)~~ - restored, bad idea
246
+ - (optional) pausing upon processing failures ```pause_timeout```
247
+ - Karafka console main process no longer intercepts irb errors
248
+ - Wiki updates
249
+ - #204 - Long running controllers
250
+ - Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
251
+ - #207 - Rename before_enqueued to after_received
252
+ - #147 - De-attach Karafka from Sidekiq by extracting Sidekiq backend
253
+
254
+ ### New features and improvements
255
+
256
+ - batch processing thanks to ```#batch_consuming``` flag and ```#params_batch``` on controllers
257
+ - ```#topic``` method on an controller instance to make a clear distinction in between params and route details
258
+ - Changed routing model (still compatible with 0.5) to allow better resources management
259
+ - Lower memory requirements due to object creation limitation (2-3 times less objects on each new message)
260
+ - Introduced the ```#batch_consuming``` config flag (config for #126) that can be set per each consumer_group
261
+ - Added support for partition, offset and partition key in the params hash
262
+ - ```name``` option in config renamed to ```client_id```
263
+ - Long running controllers with ```persistent``` flag on a topic config level, to make controller instances persistent between messages batches (single controller instance per topic per partition no per messages batch) - turned on by default
264
+
265
+ ### Incompatibilities
266
+
267
+ - Default boot file is renamed from app.rb to karafka.rb
268
+ - Removed worker glass as dependency (now and independent gem)
269
+ - ```kafka.hosts``` option renamed to ```kafka.seed_brokers``` - you don't need to provide all the hosts to work with Kafka
270
+ - ```start_from_beginning``` moved into kafka scope (```kafka.start_from_beginning```)
271
+ - Router no longer checks for route uniqueness - now you can define same routes for multiple kafkas and do a lot of crazy stuff, so it's your responsibility to check uniqueness
272
+ - Change in the way we identify topics in between Karafka and Sidekiq workers. If you upgrade, please make sure, all the jobs scheduled in Sidekiq are finished before the upgrade.
273
+ - ```batch_mode``` renamed to ```batch_fetching```
274
+ - Renamed content to value to better resemble ruby-kafka internal messages naming convention
275
+ - When having a responder with ```required``` topics and not using ```#respond_with``` at all, it will raise an exception
276
+ - Renamed ```inline_mode``` to ```inline_processing``` to resemble other settings conventions
277
+ - Renamed ```inline_processing``` to ```backend``` to reach 1.0 future compatibility
278
+ - Single controller **needs** to be used for a single topic consumption
279
+ - Renamed ```before_enqueue``` to ```after_received``` to better resemble internal logic, since for inline backend, there is no enqueue.
280
+ - Due to the level on which topic and controller are related (class level), the dynamic worker selection is no longer available.
281
+ - Renamed params #retrieve to params #retrieve! to better reflect what it does
282
+
283
+ ### Other changes
284
+ - PolishGeeksDevTools removed (in favour of Coditsu)
285
+ - Waaaaaay better code quality thanks to switching from dev tools to Coditsu
286
+ - Gem bump
287
+ - Cleaner internal API
288
+ - SRP
289
+ - Better settings proxying and management between ruby-kafka and karafka
290
+ - All internal validations are now powered by dry-validation
291
+ - Better naming conventions to reflect Kafka reality
292
+ - Removed Karafka::Connection::Message in favour of direct message details extraction from Kafka::FetchedMessage
293
+
294
+ ## 0.5.0.3
295
+ - #132 - When Kafka is gone, should reconnect after a time period
296
+ - #136 - new ruby-kafka version + other gem bumps
297
+ - ruby-kafka update
298
+ - #135 - NonMatchingRouteError - better error description in the code
299
+ - #140 - Move Capistrano Karafka to a different specific gem
300
+ - #110 - Add call method on a responder class to alias instance build and call
301
+ - #76 - Configs validator
302
+ - #138 - Possibility to have no worker class defined if inline_mode is being used
303
+ - #145 - Topic Mapper
304
+ - Ruby update to 2.4.1
305
+ - Gem bump x2
306
+ - #158 - Update docs section on heroku usage
307
+ - #150 - Add support for start_from_beginning on a per topic basis
308
+ - #148 - Lower Karafka Sidekiq dependency
309
+ - Allow karafka root to be specified from ENV
310
+ - Handle SIGTERM as a shutdown command for kafka server to support Heroku deployment
311
+
312
+ ## 0.5.0.2
313
+ - Gems update x3
314
+ - Default Ruby set to 2.3.3
315
+ - ~~Default Ruby set to 2.4.0~~
316
+ - Readme updates to match bug fixes and resolved issues
317
+ - #95 - Allow options into responder
318
+ - #98 - Use parser when responding on a topic
319
+ - #114 - Option to configure waterdrop connection pool timeout and concurrency
320
+ - #118 - Added dot in topic validation format
321
+ - #119 - add support for authentication using SSL
322
+ - #121 - JSON as a default for standalone responders usage
323
+ - #122 - Allow on capistrano role customization
324
+ - #125 - Add support to batch incoming messages
325
+ - #130 - start_from_beginning flag on routes and default
326
+ - #128 - Monitor caller_label not working with super on inheritance
327
+ - Renamed *inline* to *inline_mode* to stay consistent with flags that change the way karafka works (#125)
328
+ - Dry-configurable bump to 0.5 with fixed proc value evaluation on retrieve patch (internal change)
329
+
330
+ ## 0.5.0.1
331
+ - Fixed inconsistency in responders non-required topic definition. Now only required: false available
332
+ - #101 - Responders fail when multiple_usage true and required false
333
+ - fix error on startup from waterdrop #102
334
+ - Waterdrop 0.3.2.1 with kafka.hosts instead of kafka_hosts
335
+ - #105 - Karafka::Monitor#caller_label not working with inherited monitors
336
+ - #99 - Standalone mode (without Sidekiq)
337
+ - #97 - Buffer responders single topics before send (pre-validation)
338
+ - Better control over consumer thanks to additional config options
339
+ - #111 - Dynamic worker assignment based on the income params
340
+ - Long shutdown time fix
341
+
342
+ ## 0.5.0
343
+ - Removed Zookeeper totally as dependency
344
+ - Better group and partition rebalancing
345
+ - Automatic thread management (no need for tuning) - each topic is a separate actor/thread
346
+ - Moved from Poseidon into Ruby-Kafka
347
+ - No more max_concurrency setting
348
+ - After you define your App class and routes (and everything else) you need to add execute App.boot!
349
+ - Manual consuming is no longer available (no more karafka consume command)
350
+ - Karafka topics CLI is no longer available. No Zookeeper - no global topic discovery
351
+ - Dropped ZK as dependency
352
+ - karafka info command no longer prints details about Zookeeper
353
+ - Better shutdown
354
+ - No more autodiscovery via Zookeeper - instead, the whole cluster will be discovered directly from Kafka
355
+ - No more support for Kafka 0.8
356
+ - Support for Kafka 0.9
357
+ - No more need for ActorCluster, since now we have a single thread (and Kafka connection) per topic
358
+ - Ruby 2.2.* support dropped
359
+ - Using App name as a Kafka client_id
360
+ - Automatic Capistrano integration
361
+ - Responders support for handling better responses pipe-lining and better responses flow description and design (see README for more details)
362
+ - Gem bump
363
+ - Readme updates
364
+ - karafka flow CLI command for printing the application flow
365
+ - Some internal refactoring
366
+
367
+ ## 0.4.2
368
+ - #87 - Re-consume mode with crone for better Rails/Rack integration
369
+ - Moved Karafka server related stuff into separate Karafka::Server class
370
+ - Renamed Karafka::Runner into Karafka::Fetcher
371
+ - Gem bump
372
+ - Added chroot option to Zookeeper options
373
+ - Moved BROKERS_PATH into config from constant
374
+ - Added Karafka consume CLI action for a short running single consumption round
375
+ - Small fixes to close broken connections
376
+ - Readme updates
377
+
378
+ ## 0.4.1
379
+ - Explicit throw(:abort) required to halt before_enqueue (like in Rails 5)
380
+ - #61 - autodiscovery of Kafka brokers based on Zookeeper data
381
+ - #63 - Graceful shutdown with current offset state during data processing
382
+ - #65 - Example of NewRelic monitor is outdated
383
+ - #71 - Setup should be executed after user code is loaded
384
+ - Gem bump x3
385
+ - Rubocop remarks
386
+ - worker_timeout config option has been removed. It now needs to be defined manually by the framework user because WorkerGlass::Timeout can be disabled and we cannot use Karafka settings on a class level to initialize user code stuff
387
+ - Moved setup logic under setup/Setup namespace
388
+ - Better defaults handling
389
+ - #75 - Kafka and Zookeeper options as a hash
390
+ - #82 - Karafka autodiscovery fails upon caching of configs
391
+ - #81 - Switch config management to dry configurable
392
+ - Version fix
393
+ - Dropped support for Ruby 2.1.*
394
+ - Ruby bump to 2.3.1
395
+
396
+ ## 0.4.0
397
+ - Added WaterDrop gem with default configuration
398
+ - Refactoring of config logic to simplify adding new dependencies that need to be configured based on #setup data
399
+ - Gem bump
400
+ - Readme updates
401
+ - Renamed cluster to actor_cluster for method names
402
+ - Replaced SidekiqGlass with generic WorkerGlass lib
403
+ - Application bootstrap in app.rb no longer required
404
+ - Karafka.boot needs to be executed after all the application files are loaded (template updated)
405
+ - Small loader refactor (no API changes)
406
+ - Ruby 2.3.0 support (default)
407
+ - No more rake tasks
408
+ - Karafka CLI instead of rake tasks
409
+ - Worker cli command allows passing additional options directly to Sidekiq
410
+ - Renamed concurrency to max_concurrency - it describes better what happens - Karafka will use this number of threads only when required
411
+ - Added wait_timeout that allows us to tune how long should we wait on a single socket connection (single topic) for new messages before going to next one (this applies to each thread separately)
412
+ - Rubocop remarks
413
+ - Removed Sinatra and Puma dependencies
414
+ - Karafka Cli internal reorganization
415
+ - Karafka Cli routes task
416
+ - #37 - warn log for failed parsing of a message
417
+ - #43 - wrong constant name
418
+ - #44 - Method name conflict
419
+ - #48 - Cannot load such file -- celluloid/current
420
+ - #46 - Loading application
421
+ - #45 - Set up monitor in config
422
+ - #47 - rake karafka:run uses app.rb only
423
+ - #53 - README update with Sinatra/Rails integration description
424
+ - #41 - New Routing engine
425
+ - #54 - Move Karafka::Workers::BaseWorker to Karafka::BaseWorker
426
+ - #55 - ApplicationController and ApplicationWorker
427
+
428
+ ## 0.3.2
429
+ - Karafka::Params::Params lazy load merge keys with string/symbol names priorities fix
430
+
431
+ ## 0.3.1
432
+ - Renamed Karafka::Monitor to Karafka::Process to represent a Karafka process wrapper
433
+ - Added Karafka::Monitoring that allows to add custom logging and monitoring with external libraries and systems
434
+ - Moved logging functionality into Karafka::Monitoring default monitoring
435
+ - Added possibility to provide own monitoring as long as in responds to #notice and #notice_error
436
+ - Standardized logging format for all logs
437
+
438
+ ## 0.3.0
439
+ - Switched from custom ParserError for each parser to general catching of Karafka::Errors::ParseError and its descendants
440
+ - Gem bump
441
+ - Fixed #32 - now when using custom workers that does not inherit from Karafka::BaseWorker perform method is not required. Using custom workers means that the logic that would normally lie under #perform, needs to be executed directly from the worker.
442
+ - Fixed #31 - Technically didn't fix because this is how Sidekiq is meant to work, but provided possibility to assign custom interchangers that allow to bypass JSON encoding issues by converting data that goes to Redis to a required format (and parsing it back when it is fetched)
443
+ - Added full parameters lazy load - content is no longer loaded during #perform_async if params are not used in before_enqueue
444
+ - No more namespaces for Redis by default (use separate DBs)
445
+
446
+ ## 0.1.21
447
+ - Sidekiq 4.0.1 bump
448
+ - Gem bump
449
+ - Added direct celluloid requirement to Karafka (removed from Sidekiq)
450
+
451
+ ## 0.1.19
452
+ - Internal call - schedule naming change
453
+ - Enqueue to perform_async naming in controller to follow Sidekiq naming convention
454
+ - Gem bump
455
+
456
+ ## 0.1.18
457
+ - Changed Redis configuration options into a single hash that is directly passed to Redis setup for Sidekiq
458
+ - Added config.ru to provide a Sidekiq web UI (see README for more details)
459
+
460
+ ## 0.1.17
461
+ - Changed Karafka::Connection::Cluster tp Karafka::Connection::ActorCluster to distinguish between a single thread actor cluster for multiple topic connection and a future feature that will allow process clusterization.
462
+ - Add an ability to use user-defined parsers for a messages
463
+ - Lazy load params for before callbacks
464
+ - Automatic loading/initializing all workers classes during startup (so Sidekiq won't fail with unknown workers exception)
465
+ - Params are now private to controller
466
+ - Added bootstrap method to app.rb
467
+
468
+ ## 0.1.16
469
+ - Cluster level error catching for all exceptions so actor is not killer
470
+ - Cluster level error logging
471
+ - Listener refactoring (QueueConsumer extracted)
472
+ - Karafka::Connection::QueueConsumer to wrap around fetching logic - technically we could replace Kafka with any other messaging engine as long as we preserve the same API
473
+ - Added debug env for debugging purpose in applications
474
+
475
+ ## 0.1.15
476
+ - Fixed max_wait_ms vs socket_timeout_ms issue
477
+ - Fixed closing queue connection after Poseidon::Errors::ProtocolError failure
478
+ - Fixed wrong logging file selection based on env
479
+ - Extracted Karafka::Connection::QueueConsumer object to wrap around queue connection
480
+
481
+ ## 0.1.14
482
+ - Rake tasks for listing all the topics on Kafka server (rake kafka:topics)
483
+
484
+ ## 0.1.13
485
+ - Ability to assign custom workers and use them bypassing Karafka::BaseWorker (or its descendants)
486
+ - Gem bump
487
+
488
+ ## 0.1.12
489
+ - All internal errors went to Karafka::Errors namespace
490
+
491
+ ## 0.1.11
492
+ - Rescuing all the "before Sidekiq" processing so errors won't affect other incoming messages
493
+ - Fixed dying actors after connection error
494
+ - Added a new app status - "initializing"
495
+ - Karafka::Status model cleanup
496
+
497
+ ## 0.1.10
498
+ - Added possibility to specify redis namespace in configuration (failover to app name)
499
+ - Renamed redis_host to redis_url in configuration
500
+
501
+ ## 0.1.9
502
+ - Added worker logger
503
+
504
+ ## 0.1.8
505
+ - Dropped local env support in favour of [Envlogic](https://github.com/karafka/envlogic) - no changes in API
506
+
507
+ ## 0.1.7
508
+ - Karafka option for Redis hosts (not localhost only)
509
+
510
+ ## 0.1.6
511
+ - Added better concurency by clusterization of listeners
512
+ - Added graceful shutdown
513
+ - Added concurency that allows to handle bigger applications with celluloid
514
+ - Karafka controllers no longer require group to be defined (created based on the topic and app name)
515
+ - Karafka controllers no longer require topic to be defined (created based on the controller name)
516
+ - Readme updates
517
+
518
+ ## 0.1.5
519
+ - Celluloid support for listeners
520
+ - Multi target logging (STDOUT and file)
521
+
522
+ ## 0.1.4
523
+ - Renamed events to messages to follow Apache Kafka naming convention
524
+
525
+ ## 0.1.3
526
+ - Karafka::App.logger moved to Karafka.logger
527
+ - README updates (Usage section was added)
528
+
529
+ ## 0.1.2
530
+ - Logging to log/environment.log
531
+ - Karafka::Runner
532
+
533
+ ## 0.1.1
534
+ - README updates
535
+ - Rake tasks updates
536
+ - Rake installation task
537
+ - Changelog file added
538
+
539
+ ## 0.1.0
540
+ - Initial framework code