karafka 1.4.15 → 2.0.0.alpha1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/FUNDING.yml +3 -0
  4. data/.github/workflows/ci.yml +74 -24
  5. data/.ruby-version +1 -1
  6. data/CHANGELOG.md +38 -39
  7. data/Gemfile +6 -0
  8. data/Gemfile.lock +50 -52
  9. data/LICENSE +14 -0
  10. data/LICENSE-COMM +89 -0
  11. data/LICENSE-LGPL +165 -0
  12. data/README.md +59 -14
  13. data/bin/benchmarks +85 -0
  14. data/bin/create_token +28 -0
  15. data/bin/integrations +160 -0
  16. data/bin/stress +13 -0
  17. data/certs/karafka-pro.pem +11 -0
  18. data/certs/mensfeld.pem +23 -24
  19. data/config/errors.yml +4 -38
  20. data/docker-compose.yml +11 -3
  21. data/karafka.gemspec +10 -20
  22. data/lib/active_job/consumer.rb +22 -0
  23. data/lib/active_job/karafka.rb +18 -0
  24. data/lib/active_job/queue_adapters/karafka_adapter.rb +29 -0
  25. data/lib/active_job/routing_extensions.rb +15 -0
  26. data/lib/karafka/app.rb +13 -20
  27. data/lib/karafka/base_consumer.rb +103 -34
  28. data/lib/karafka/cli/base.rb +4 -4
  29. data/lib/karafka/cli/info.rb +43 -8
  30. data/lib/karafka/cli/install.rb +3 -8
  31. data/lib/karafka/cli/server.rb +17 -30
  32. data/lib/karafka/cli.rb +4 -11
  33. data/lib/karafka/connection/client.rb +279 -93
  34. data/lib/karafka/connection/listener.rb +137 -38
  35. data/lib/karafka/connection/messages_buffer.rb +57 -0
  36. data/lib/karafka/connection/pauses_manager.rb +46 -0
  37. data/lib/karafka/connection/rebalance_manager.rb +62 -0
  38. data/lib/karafka/contracts/config.rb +25 -7
  39. data/lib/karafka/contracts/consumer_group.rb +0 -173
  40. data/lib/karafka/contracts/consumer_group_topic.rb +17 -7
  41. data/lib/karafka/contracts/server_cli_options.rb +1 -9
  42. data/lib/karafka/contracts.rb +1 -1
  43. data/lib/karafka/env.rb +46 -0
  44. data/lib/karafka/errors.rb +14 -18
  45. data/lib/karafka/helpers/multi_delegator.rb +2 -2
  46. data/lib/karafka/instrumentation/callbacks/error.rb +40 -0
  47. data/lib/karafka/instrumentation/callbacks/statistics.rb +42 -0
  48. data/lib/karafka/instrumentation/monitor.rb +14 -21
  49. data/lib/karafka/instrumentation/stdout_listener.rb +64 -91
  50. data/lib/karafka/instrumentation.rb +21 -0
  51. data/lib/karafka/licenser.rb +65 -0
  52. data/lib/karafka/{params → messages}/batch_metadata.rb +7 -13
  53. data/lib/karafka/messages/builders/batch_metadata.rb +30 -0
  54. data/lib/karafka/messages/builders/message.rb +38 -0
  55. data/lib/karafka/messages/builders/messages.rb +40 -0
  56. data/lib/karafka/{params/params.rb → messages/message.rb} +7 -12
  57. data/lib/karafka/messages/messages.rb +64 -0
  58. data/lib/karafka/{params → messages}/metadata.rb +4 -6
  59. data/lib/karafka/messages/seek.rb +9 -0
  60. data/lib/karafka/patches/rdkafka/consumer.rb +22 -0
  61. data/lib/karafka/processing/executor.rb +96 -0
  62. data/lib/karafka/processing/executors_buffer.rb +49 -0
  63. data/lib/karafka/processing/jobs/base.rb +18 -0
  64. data/lib/karafka/processing/jobs/consume.rb +28 -0
  65. data/lib/karafka/processing/jobs/revoked.rb +22 -0
  66. data/lib/karafka/processing/jobs/shutdown.rb +23 -0
  67. data/lib/karafka/processing/jobs_queue.rb +121 -0
  68. data/lib/karafka/processing/worker.rb +57 -0
  69. data/lib/karafka/processing/workers_batch.rb +22 -0
  70. data/lib/karafka/railtie.rb +65 -0
  71. data/lib/karafka/routing/builder.rb +15 -14
  72. data/lib/karafka/routing/consumer_group.rb +10 -18
  73. data/lib/karafka/routing/consumer_mapper.rb +1 -2
  74. data/lib/karafka/routing/router.rb +1 -1
  75. data/lib/karafka/routing/subscription_group.rb +53 -0
  76. data/lib/karafka/routing/subscription_groups_builder.rb +51 -0
  77. data/lib/karafka/routing/topic.rb +47 -25
  78. data/lib/karafka/runner.rb +59 -0
  79. data/lib/karafka/serialization/json/deserializer.rb +6 -15
  80. data/lib/karafka/server.rb +62 -25
  81. data/lib/karafka/setup/config.rb +86 -159
  82. data/lib/karafka/status.rb +13 -3
  83. data/lib/karafka/templates/example_consumer.rb.erb +16 -0
  84. data/lib/karafka/templates/karafka.rb.erb +14 -50
  85. data/lib/karafka/time_trackers/base.rb +19 -0
  86. data/lib/karafka/time_trackers/pause.rb +84 -0
  87. data/lib/karafka/time_trackers/poll.rb +65 -0
  88. data/lib/karafka/version.rb +1 -1
  89. data/lib/karafka.rb +30 -44
  90. data.tar.gz.sig +0 -0
  91. metadata +96 -132
  92. metadata.gz.sig +0 -0
  93. data/MIT-LICENCE +0 -18
  94. data/lib/karafka/assignment_strategies/round_robin.rb +0 -13
  95. data/lib/karafka/attributes_map.rb +0 -63
  96. data/lib/karafka/backends/inline.rb +0 -16
  97. data/lib/karafka/base_responder.rb +0 -226
  98. data/lib/karafka/cli/flow.rb +0 -48
  99. data/lib/karafka/cli/missingno.rb +0 -19
  100. data/lib/karafka/code_reloader.rb +0 -67
  101. data/lib/karafka/connection/api_adapter.rb +0 -158
  102. data/lib/karafka/connection/batch_delegator.rb +0 -55
  103. data/lib/karafka/connection/builder.rb +0 -23
  104. data/lib/karafka/connection/message_delegator.rb +0 -36
  105. data/lib/karafka/consumers/batch_metadata.rb +0 -10
  106. data/lib/karafka/consumers/callbacks.rb +0 -71
  107. data/lib/karafka/consumers/includer.rb +0 -64
  108. data/lib/karafka/consumers/responders.rb +0 -24
  109. data/lib/karafka/consumers/single_params.rb +0 -15
  110. data/lib/karafka/contracts/responder_usage.rb +0 -54
  111. data/lib/karafka/fetcher.rb +0 -42
  112. data/lib/karafka/helpers/class_matcher.rb +0 -88
  113. data/lib/karafka/helpers/config_retriever.rb +0 -46
  114. data/lib/karafka/helpers/inflector.rb +0 -26
  115. data/lib/karafka/params/builders/batch_metadata.rb +0 -30
  116. data/lib/karafka/params/builders/params.rb +0 -38
  117. data/lib/karafka/params/builders/params_batch.rb +0 -25
  118. data/lib/karafka/params/params_batch.rb +0 -60
  119. data/lib/karafka/patches/ruby_kafka.rb +0 -47
  120. data/lib/karafka/persistence/client.rb +0 -29
  121. data/lib/karafka/persistence/consumers.rb +0 -45
  122. data/lib/karafka/persistence/topics.rb +0 -48
  123. data/lib/karafka/responders/builder.rb +0 -36
  124. data/lib/karafka/responders/topic.rb +0 -55
  125. data/lib/karafka/routing/topic_mapper.rb +0 -53
  126. data/lib/karafka/serialization/json/serializer.rb +0 -31
  127. data/lib/karafka/setup/configurators/water_drop.rb +0 -36
  128. data/lib/karafka/templates/application_responder.rb.erb +0 -11
data/lib/karafka.rb CHANGED
@@ -3,63 +3,48 @@
3
3
  %w[
4
4
  delegate
5
5
  English
6
+ rdkafka
6
7
  waterdrop
7
- kafka
8
- envlogic
9
8
  json
10
9
  thor
11
10
  forwardable
12
11
  fileutils
13
- concurrent
12
+ openssl
13
+ base64
14
+ date
14
15
  dry-configurable
15
16
  dry-validation
16
17
  dry/events/publisher
17
- dry/inflector
18
18
  dry/monitor/notifications
19
- dry/core/constants
20
19
  zeitwerk
21
20
  ].each(&method(:require))
22
21
 
23
- unless ENV.key?('I_ACCEPT_CRITICAL_ERRORS_IN_KARAFKA_1_4')
24
- 3.times { puts }
25
- puts '~' * 50
26
- puts "~~~ \033[31mCRITICAL\033[0m NOTICE ON KARAFKA 1.4 reliability ~~~"
27
- puts '~' * 50
28
- puts
29
- puts 'Karafka 1.4 is no longer supported and contains critical errors, included but not limited to:'
30
- puts
31
- puts ' - Double processing of messages'
32
- puts ' - Skipping messages'
33
- puts ' - Hanging during processing'
34
- puts ' - Unexpectedly stopping message processing'
35
- puts ' - Failure to deliver messages to Kafka'
36
- puts ' - Resetting the consumer group and starting from the beginning'
37
- puts
38
- puts 'To resolve these issues, it is highly recommended to upgrade to Karafka 2.1 or higher.'
39
- puts
40
- puts 'If you want to ignore this message and continue, set the I_ACCEPT_CRITICAL_ERRORS_IN_KARAFKA_1_4 env variable to true.'
41
- puts
42
- puts 'Apologies for any inconvenience caused by this release.'
43
- puts 'There is no other way to make sure, that you are notified about those issues and their severity.'
44
- puts
45
- puts 'If you need help with the upgrade, we do have a Slack channel you can join: https://slack.karafka.io/'
46
- puts
47
- puts '~' * 50
48
- puts
49
-
50
- raise RuntimeError
51
- end
52
-
53
- # Karafka library
22
+ # Karafka framework main namespace
54
23
  module Karafka
55
- extend Envlogic
56
-
57
24
  class << self
25
+ # @return [Karafka::Env] env instance that allows us to check environment
26
+ def env
27
+ @env ||= Env.new
28
+ end
29
+
30
+ # @param environment [String, Symbol] new environment that we want to set
31
+ # @return [Karafka::Env] env instance
32
+ # @example Assign new environment to Karafka::App
33
+ # Karafka::App.env = :production
34
+ def env=(environment)
35
+ env.replace(environment.to_s)
36
+ end
37
+
58
38
  # @return [Logger] logger that we want to use. Will use ::Karafka::Logger by default
59
39
  def logger
60
40
  @logger ||= App.config.logger
61
41
  end
62
42
 
43
+ # @return [WaterDrop::Producer] waterdrop messages producer
44
+ def producer
45
+ @producer ||= App.config.producer
46
+ end
47
+
63
48
  # @return [::Karafka::Monitor] monitor that we want to use
64
49
  def monitor
65
50
  @monitor ||= App.config.monitor
@@ -95,9 +80,10 @@ module Karafka
95
80
  end
96
81
  end
97
82
 
98
- Zeitwerk::Loader
99
- .for_gem
100
- .tap(&:setup)
101
- .tap(&:eager_load)
102
-
103
- Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
83
+ loader = Zeitwerk::Loader.for_gem
84
+ # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
85
+ loader.do_not_eager_load(Karafka.gem_root.join('lib/active_job'))
86
+ # Do not load pro components, this will be handled by license manager
87
+ loader.do_not_eager_load(Karafka.gem_root.join('lib/karafka/pro'))
88
+ loader.setup
89
+ loader.eager_load
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,86 +1,55 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: karafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.15
4
+ version: 2.0.0.alpha1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
8
- - Pavlo Vavruk
9
- - Adam Gwozdowski
10
8
  autorequire:
11
9
  bindir: bin
12
10
  cert_chain:
13
11
  - |
14
12
  -----BEGIN CERTIFICATE-----
15
- MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
16
- YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
17
- MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
18
- dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
19
- bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
20
- jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
21
- F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
22
- ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
23
- lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
24
- rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
25
- KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
26
- Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
27
- cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
28
- HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
29
- iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
30
- bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
31
- G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
32
- vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
33
- yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
34
- EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
35
- Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
36
- KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
37
- Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
38
- MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
13
+ MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
14
+ ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
15
+ NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
16
+ CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
17
+ ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
18
+ TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
19
+ 1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
20
+ vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
21
+ qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
22
+ tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
23
+ g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
24
+ LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
25
+ BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
26
+ EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
27
+ c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
28
+ W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
29
+ yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
30
+ 8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
31
+ /rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
32
+ fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
33
+ 2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
34
+ R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
35
+ pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
39
36
  -----END CERTIFICATE-----
40
- date: 2023-06-19 00:00:00.000000000 Z
37
+ date: 2022-01-30 00:00:00.000000000 Z
41
38
  dependencies:
42
- - !ruby/object:Gem::Dependency
43
- name: concurrent-ruby
44
- requirement: !ruby/object:Gem::Requirement
45
- requirements:
46
- - - ">="
47
- - !ruby/object:Gem::Version
48
- version: '0'
49
- type: :runtime
50
- prerelease: false
51
- version_requirements: !ruby/object:Gem::Requirement
52
- requirements:
53
- - - ">="
54
- - !ruby/object:Gem::Version
55
- version: '0'
56
39
  - !ruby/object:Gem::Dependency
57
40
  name: dry-configurable
58
41
  requirement: !ruby/object:Gem::Requirement
59
42
  requirements:
60
43
  - - "~>"
61
44
  - !ruby/object:Gem::Version
62
- version: '0.16'
63
- type: :runtime
64
- prerelease: false
65
- version_requirements: !ruby/object:Gem::Requirement
66
- requirements:
67
- - - "~>"
68
- - !ruby/object:Gem::Version
69
- version: '0.16'
70
- - !ruby/object:Gem::Dependency
71
- name: dry-inflector
72
- requirement: !ruby/object:Gem::Requirement
73
- requirements:
74
- - - "~>"
75
- - !ruby/object:Gem::Version
76
- version: '0.2'
45
+ version: '0.13'
77
46
  type: :runtime
78
47
  prerelease: false
79
48
  version_requirements: !ruby/object:Gem::Requirement
80
49
  requirements:
81
50
  - - "~>"
82
51
  - !ruby/object:Gem::Version
83
- version: '0.2'
52
+ version: '0.13'
84
53
  - !ruby/object:Gem::Dependency
85
54
  name: dry-monitor
86
55
  requirement: !ruby/object:Gem::Requirement
@@ -110,80 +79,70 @@ dependencies:
110
79
  - !ruby/object:Gem::Version
111
80
  version: '1.7'
112
81
  - !ruby/object:Gem::Dependency
113
- name: envlogic
114
- requirement: !ruby/object:Gem::Requirement
115
- requirements:
116
- - - "~>"
117
- - !ruby/object:Gem::Version
118
- version: '1.1'
119
- type: :runtime
120
- prerelease: false
121
- version_requirements: !ruby/object:Gem::Requirement
122
- requirements:
123
- - - "~>"
124
- - !ruby/object:Gem::Version
125
- version: '1.1'
126
- - !ruby/object:Gem::Dependency
127
- name: ruby-kafka
82
+ name: rdkafka
128
83
  requirement: !ruby/object:Gem::Requirement
129
84
  requirements:
130
85
  - - ">="
131
86
  - !ruby/object:Gem::Version
132
- version: 1.3.0
87
+ version: '0.10'
133
88
  type: :runtime
134
89
  prerelease: false
135
90
  version_requirements: !ruby/object:Gem::Requirement
136
91
  requirements:
137
92
  - - ">="
138
93
  - !ruby/object:Gem::Version
139
- version: 1.3.0
94
+ version: '0.10'
140
95
  - !ruby/object:Gem::Dependency
141
96
  name: thor
142
97
  requirement: !ruby/object:Gem::Requirement
143
98
  requirements:
144
99
  - - ">="
145
100
  - !ruby/object:Gem::Version
146
- version: '1.1'
101
+ version: '0.20'
147
102
  type: :runtime
148
103
  prerelease: false
149
104
  version_requirements: !ruby/object:Gem::Requirement
150
105
  requirements:
151
106
  - - ">="
152
107
  - !ruby/object:Gem::Version
153
- version: '1.1'
108
+ version: '0.20'
154
109
  - !ruby/object:Gem::Dependency
155
110
  name: waterdrop
156
111
  requirement: !ruby/object:Gem::Requirement
157
112
  requirements:
158
- - - "~>"
113
+ - - ">="
159
114
  - !ruby/object:Gem::Version
160
- version: '1.4'
115
+ version: 2.1.0
116
+ - - "<"
117
+ - !ruby/object:Gem::Version
118
+ version: 3.0.0
161
119
  type: :runtime
162
120
  prerelease: false
163
121
  version_requirements: !ruby/object:Gem::Requirement
164
122
  requirements:
165
- - - "~>"
123
+ - - ">="
166
124
  - !ruby/object:Gem::Version
167
- version: '1.4'
125
+ version: 2.1.0
126
+ - - "<"
127
+ - !ruby/object:Gem::Version
128
+ version: 3.0.0
168
129
  - !ruby/object:Gem::Dependency
169
130
  name: zeitwerk
170
131
  requirement: !ruby/object:Gem::Requirement
171
132
  requirements:
172
133
  - - "~>"
173
134
  - !ruby/object:Gem::Version
174
- version: '2.6'
135
+ version: '2.3'
175
136
  type: :runtime
176
137
  prerelease: false
177
138
  version_requirements: !ruby/object:Gem::Requirement
178
139
  requirements:
179
140
  - - "~>"
180
141
  - !ruby/object:Gem::Version
181
- version: '2.6'
142
+ version: '2.3'
182
143
  description: Framework used to simplify Apache Kafka based Ruby applications development
183
144
  email:
184
145
  - maciej@mensfeld.pl
185
- - pavlo.vavruk@gmail.com
186
- - adam99g@gmail.com
187
146
  executables:
188
147
  - karafka
189
148
  extensions: []
@@ -192,6 +151,7 @@ files:
192
151
  - ".coditsu/ci.yml"
193
152
  - ".console_irbrc"
194
153
  - ".diffend.yml"
154
+ - ".github/FUNDING.yml"
195
155
  - ".github/ISSUE_TEMPLATE/bug_report.md"
196
156
  - ".github/ISSUE_TEMPLATE/feature_request.md"
197
157
  - ".github/workflows/ci.yml"
@@ -204,99 +164,103 @@ files:
204
164
  - CONTRIBUTING.md
205
165
  - Gemfile
206
166
  - Gemfile.lock
207
- - MIT-LICENCE
167
+ - LICENSE
168
+ - LICENSE-COMM
169
+ - LICENSE-LGPL
208
170
  - README.md
171
+ - bin/benchmarks
172
+ - bin/create_token
173
+ - bin/integrations
209
174
  - bin/karafka
175
+ - bin/stress
176
+ - certs/karafka-pro.pem
210
177
  - certs/mensfeld.pem
211
178
  - config/errors.yml
212
179
  - docker-compose.yml
213
180
  - karafka.gemspec
181
+ - lib/active_job/consumer.rb
182
+ - lib/active_job/karafka.rb
183
+ - lib/active_job/queue_adapters/karafka_adapter.rb
184
+ - lib/active_job/routing_extensions.rb
214
185
  - lib/karafka.rb
215
186
  - lib/karafka/app.rb
216
- - lib/karafka/assignment_strategies/round_robin.rb
217
- - lib/karafka/attributes_map.rb
218
- - lib/karafka/backends/inline.rb
219
187
  - lib/karafka/base_consumer.rb
220
- - lib/karafka/base_responder.rb
221
188
  - lib/karafka/cli.rb
222
189
  - lib/karafka/cli/base.rb
223
190
  - lib/karafka/cli/console.rb
224
- - lib/karafka/cli/flow.rb
225
191
  - lib/karafka/cli/info.rb
226
192
  - lib/karafka/cli/install.rb
227
- - lib/karafka/cli/missingno.rb
228
193
  - lib/karafka/cli/server.rb
229
- - lib/karafka/code_reloader.rb
230
- - lib/karafka/connection/api_adapter.rb
231
- - lib/karafka/connection/batch_delegator.rb
232
- - lib/karafka/connection/builder.rb
233
194
  - lib/karafka/connection/client.rb
234
195
  - lib/karafka/connection/listener.rb
235
- - lib/karafka/connection/message_delegator.rb
236
- - lib/karafka/consumers/batch_metadata.rb
237
- - lib/karafka/consumers/callbacks.rb
238
- - lib/karafka/consumers/includer.rb
239
- - lib/karafka/consumers/responders.rb
240
- - lib/karafka/consumers/single_params.rb
196
+ - lib/karafka/connection/messages_buffer.rb
197
+ - lib/karafka/connection/pauses_manager.rb
198
+ - lib/karafka/connection/rebalance_manager.rb
241
199
  - lib/karafka/contracts.rb
242
200
  - lib/karafka/contracts/config.rb
243
201
  - lib/karafka/contracts/consumer_group.rb
244
202
  - lib/karafka/contracts/consumer_group_topic.rb
245
- - lib/karafka/contracts/responder_usage.rb
246
203
  - lib/karafka/contracts/server_cli_options.rb
204
+ - lib/karafka/env.rb
247
205
  - lib/karafka/errors.rb
248
- - lib/karafka/fetcher.rb
249
- - lib/karafka/helpers/class_matcher.rb
250
- - lib/karafka/helpers/config_retriever.rb
251
- - lib/karafka/helpers/inflector.rb
252
206
  - lib/karafka/helpers/multi_delegator.rb
207
+ - lib/karafka/instrumentation.rb
208
+ - lib/karafka/instrumentation/callbacks/error.rb
209
+ - lib/karafka/instrumentation/callbacks/statistics.rb
253
210
  - lib/karafka/instrumentation/logger.rb
254
211
  - lib/karafka/instrumentation/monitor.rb
255
212
  - lib/karafka/instrumentation/proctitle_listener.rb
256
213
  - lib/karafka/instrumentation/stdout_listener.rb
257
- - lib/karafka/params/batch_metadata.rb
258
- - lib/karafka/params/builders/batch_metadata.rb
259
- - lib/karafka/params/builders/params.rb
260
- - lib/karafka/params/builders/params_batch.rb
261
- - lib/karafka/params/metadata.rb
262
- - lib/karafka/params/params.rb
263
- - lib/karafka/params/params_batch.rb
264
- - lib/karafka/patches/ruby_kafka.rb
265
- - lib/karafka/persistence/client.rb
266
- - lib/karafka/persistence/consumers.rb
267
- - lib/karafka/persistence/topics.rb
214
+ - lib/karafka/licenser.rb
215
+ - lib/karafka/messages/batch_metadata.rb
216
+ - lib/karafka/messages/builders/batch_metadata.rb
217
+ - lib/karafka/messages/builders/message.rb
218
+ - lib/karafka/messages/builders/messages.rb
219
+ - lib/karafka/messages/message.rb
220
+ - lib/karafka/messages/messages.rb
221
+ - lib/karafka/messages/metadata.rb
222
+ - lib/karafka/messages/seek.rb
223
+ - lib/karafka/patches/rdkafka/consumer.rb
268
224
  - lib/karafka/process.rb
269
- - lib/karafka/responders/builder.rb
270
- - lib/karafka/responders/topic.rb
225
+ - lib/karafka/processing/executor.rb
226
+ - lib/karafka/processing/executors_buffer.rb
227
+ - lib/karafka/processing/jobs/base.rb
228
+ - lib/karafka/processing/jobs/consume.rb
229
+ - lib/karafka/processing/jobs/revoked.rb
230
+ - lib/karafka/processing/jobs/shutdown.rb
231
+ - lib/karafka/processing/jobs_queue.rb
232
+ - lib/karafka/processing/worker.rb
233
+ - lib/karafka/processing/workers_batch.rb
234
+ - lib/karafka/railtie.rb
271
235
  - lib/karafka/routing/builder.rb
272
236
  - lib/karafka/routing/consumer_group.rb
273
237
  - lib/karafka/routing/consumer_mapper.rb
274
238
  - lib/karafka/routing/proxy.rb
275
239
  - lib/karafka/routing/router.rb
240
+ - lib/karafka/routing/subscription_group.rb
241
+ - lib/karafka/routing/subscription_groups_builder.rb
276
242
  - lib/karafka/routing/topic.rb
277
- - lib/karafka/routing/topic_mapper.rb
243
+ - lib/karafka/runner.rb
278
244
  - lib/karafka/serialization/json/deserializer.rb
279
- - lib/karafka/serialization/json/serializer.rb
280
245
  - lib/karafka/server.rb
281
246
  - lib/karafka/setup/config.rb
282
- - lib/karafka/setup/configurators/water_drop.rb
283
247
  - lib/karafka/setup/dsl.rb
284
248
  - lib/karafka/status.rb
285
249
  - lib/karafka/templates/application_consumer.rb.erb
286
- - lib/karafka/templates/application_responder.rb.erb
250
+ - lib/karafka/templates/example_consumer.rb.erb
287
251
  - lib/karafka/templates/karafka.rb.erb
252
+ - lib/karafka/time_trackers/base.rb
253
+ - lib/karafka/time_trackers/pause.rb
254
+ - lib/karafka/time_trackers/poll.rb
288
255
  - lib/karafka/version.rb
289
256
  - log/.gitkeep
290
257
  homepage: https://karafka.io
291
258
  licenses:
292
- - MIT
259
+ - LGPL-3.0
293
260
  metadata:
294
261
  source_code_uri: https://github.com/karafka/karafka
295
262
  rubygems_mfa_required: 'true'
296
- post_install_message: |
297
- WARN: Karafka 1.4 will reach the end of life soon.
298
- We highly recommend updating to Karafka 2.0.
299
- Visit this page for more details: https://karafka.io/docs/Versions-Lifecycle-and-EOL
263
+ post_install_message:
300
264
  rdoc_options: []
301
265
  require_paths:
302
266
  - lib
@@ -304,12 +268,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
304
268
  requirements:
305
269
  - - ">="
306
270
  - !ruby/object:Gem::Version
307
- version: '2.7'
271
+ version: 2.6.0
308
272
  required_rubygems_version: !ruby/object:Gem::Requirement
309
273
  requirements:
310
- - - ">="
274
+ - - ">"
311
275
  - !ruby/object:Gem::Version
312
- version: '0'
276
+ version: 1.3.1
313
277
  requirements: []
314
278
  rubygems_version: 3.3.4
315
279
  signing_key:
metadata.gz.sig CHANGED
Binary file
data/MIT-LICENCE DELETED
@@ -1,18 +0,0 @@
1
- Permission is hereby granted, free of charge, to any person obtaining
2
- a copy of this software and associated documentation files (the
3
- "Software"), to deal in the Software without restriction, including
4
- without limitation the rights to use, copy, modify, merge, publish,
5
- distribute, sublicense, and/or sell copies of the Software, and to
6
- permit persons to whom the Software is furnished to do so, subject to
7
- the following conditions:
8
-
9
- The above copyright notice and this permission notice shall be
10
- included in all copies or substantial portions of the Software.
11
-
12
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
13
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
14
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
15
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
16
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
17
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
18
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -1,13 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Strategies for Kafka partitions assignments
5
- module AssignmentStrategies
6
- # Standard RoundRobin strategy
7
- class RoundRobin < SimpleDelegator
8
- def initialize
9
- super(Kafka::RoundRobinAssignmentStrategy.new)
10
- end
11
- end
12
- end
13
- end
@@ -1,63 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Both Karafka and Ruby-Kafka contain a lot of settings that can be applied on multiple
5
- # levels. In Karafka that is on consumer group and on the topic level. In Ruby-Kafka it
6
- # is on consumer, subscription and consumption levels. In order to maintain an order
7
- # in managing those settings, this module was created. It contains details on what setting
8
- # where should go and which layer (both on Karafka and Ruby-Kafka) is responsible for
9
- # setting it and sending it forward
10
- # @note Settings presented here cover all the settings that are being used across Karafka
11
- module AttributesMap
12
- class << self
13
- # What settings should go where in ruby-kafka
14
- # @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
15
- # @note All other settings will be passed to Kafka.new method invocation.
16
- # All elements in this hash are just edge cases
17
- def api_adapter
18
- {
19
- consumer: %i[
20
- session_timeout offset_commit_interval offset_commit_threshold
21
- offset_retention_time heartbeat_interval fetcher_max_queue_size
22
- assignment_strategy
23
- ],
24
- subscribe: %i[start_from_beginning max_bytes_per_partition],
25
- consumption: %i[min_bytes max_bytes max_wait_time],
26
- pause: %i[pause_timeout pause_max_timeout pause_exponential_backoff],
27
- # All the options that are under kafka config namespace, but are not used
28
- # directly with kafka api, but from the Karafka user perspective, they are
29
- # still related to kafka. They should not be proxied anywhere
30
- ignored: %i[reconnect_timeout automatically_mark_as_consumed]
31
- }
32
- end
33
-
34
- # @return [Array<Symbol>] properties that can be set on a per topic level
35
- def topic
36
- (api_adapter[:subscribe] + %i[
37
- backend
38
- name
39
- deserializer
40
- responder
41
- batch_consuming
42
- ]).uniq
43
- end
44
-
45
- # @return [Array<Symbol>] properties that can be set on a per consumer group level
46
- # @note Note that there are settings directly extracted from the config kafka namespace
47
- # I did this that way, so I won't have to repeat same setting keys over and over again
48
- # Thanks to this solution, if any new setting is available for ruby-kafka, we just need
49
- # to add it to our configuration class and it will be handled automatically.
50
- def consumer_group
51
- # @note We don't ignore the api_adapter[:ignored] values as they should be ignored
52
- # only when proxying details go ruby-kafka. We use ignored fields internally in karafka
53
- ignored_settings = api_adapter[:subscribe]
54
- defined_settings = api_adapter.values.flatten
55
- karafka_settings = %i[batch_fetching]
56
-
57
- dynamically_proxied = Karafka::Setup::Config.config.kafka.to_h.keys
58
-
59
- (defined_settings + dynamically_proxied).uniq + karafka_settings - ignored_settings
60
- end
61
- end
62
- end
63
- end
@@ -1,16 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Karafka
4
- # Namespace for all different backends Karafka supports
5
- module Backends
6
- # Backend that just runs stuff asap without any scheduling
7
- module Inline
8
- private
9
-
10
- # Executes consume code immediately (without enqueuing)
11
- def process
12
- Karafka.monitor.instrument('backends.inline.process', caller: self) { consume }
13
- end
14
- end
15
- end
16
- end