karafka 1.2.8 → 1.4.0

data/Gemfile.lock

@@ -1,128 +1,139 @@
 PATH
   remote: .
   specs:
-    karafka (1.2.8)
-      activesupport (>= 4.0)
-      dry-configurable (~> 0.7)
-      dry-inflector (~> 0.1.1)
-      dry-monitor (~> 0.1)
-      dry-validation (~> 0.11)
-      envlogic (~> 1.0)
-      multi_json (>= 1.12)
+    karafka (1.4.0)
+      dry-configurable (~> 0.8)
+      dry-inflector (~> 0.1)
+      dry-monitor (~> 0.3)
+      dry-validation (~> 1.2)
+      envlogic (~> 1.1)
+      irb (~> 1.0)
       rake (>= 11.3)
-      require_all (>= 1.4)
-      ruby-kafka (>= 0.6)
-      thor (~> 0.19)
-      waterdrop (~> 1.2.4)
+      ruby-kafka (>= 1.0.0)
+      thor (>= 0.20)
+      waterdrop (~> 1.4.0)
+      zeitwerk (~> 2.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.1)
+    activesupport (6.0.3.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    benchmark-ips (2.7.2)
-    concurrent-ruby (1.0.5)
-    delivery_boy (0.2.7)
-      king_konf (~> 0.2)
-      ruby-kafka (~> 0.5)
-    diff-lcs (1.3)
-    digest-crc (0.4.1)
-    docile (1.3.1)
-    dry-configurable (0.7.0)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.7)
+    delivery_boy (1.0.1)
+      king_konf (~> 0.3)
+      ruby-kafka (~> 1.0)
+    diff-lcs (1.4.4)
+    digest-crc (0.6.1)
+      rake (~> 13.0)
+    docile (1.3.2)
+    dry-configurable (0.11.6)
       concurrent-ruby (~> 1.0)
-    dry-container (0.6.0)
+      dry-core (~> 0.4, >= 0.4.7)
+      dry-equalizer (~> 0.2)
+    dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
       dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.7)
+    dry-core (0.4.9)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.1)
-    dry-events (0.1.0)
+    dry-equalizer (0.3.0)
+    dry-events (0.2.0)
       concurrent-ruby (~> 1.0)
       dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-    dry-inflector (0.1.2)
-    dry-logic (0.4.2)
-      dry-container (~> 0.2, >= 0.2.6)
+    dry-inflector (0.2.0)
+    dry-initializer (3.0.3)
+    dry-logic (1.0.7)
+      concurrent-ruby (~> 1.0)
       dry-core (~> 0.2)
       dry-equalizer (~> 0.2)
-    dry-monitor (0.1.2)
+    dry-monitor (0.3.2)
       dry-configurable (~> 0.5)
+      dry-core (~> 0.4)
+      dry-equalizer (~> 0.2)
+      dry-events (~> 0.2)
+    dry-schema (1.5.4)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 0.8, >= 0.8.3)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-events (~> 0.1)
-      rouge (~> 2.0, >= 2.2.1)
-    dry-types (0.13.2)
+      dry-initializer (~> 3.0)
+      dry-logic (~> 1.0)
+      dry-types (~> 1.4)
+    dry-types (1.4.0)
       concurrent-ruby (~> 1.0)
       dry-container (~> 0.3)
       dry-core (~> 0.4, >= 0.4.4)
-      dry-equalizer (~> 0.2)
+      dry-equalizer (~> 0.3)
       dry-inflector (~> 0.1, >= 0.1.2)
-      dry-logic (~> 0.4, >= 0.4.2)
-    dry-validation (0.12.2)
+      dry-logic (~> 1.0, >= 1.0.2)
+    dry-validation (1.5.6)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-      dry-core (~> 0.2, >= 0.2.1)
+      dry-container (~> 0.7, >= 0.7.1)
+      dry-core (~> 0.4)
       dry-equalizer (~> 0.2)
-      dry-logic (~> 0.4, >= 0.4.0)
-      dry-types (~> 0.13.1)
-    envlogic (1.1.0)
+      dry-initializer (~> 3.0)
+      dry-schema (~> 1.5, >= 1.5.2)
+    envlogic (1.1.2)
       dry-inflector (~> 0.1)
-    i18n (1.1.0)
+    factory_bot (6.1.0)
+      activesupport (>= 5.0.0)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
-    json (2.1.0-java)
-    king_konf (0.3.6)
-    minitest (5.11.3)
-    multi_json (1.13.1)
-    null-logger (0.1.5)
-    rake (12.3.1)
-    require_all (2.0.0)
-    rouge (2.2.1)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.1)
+    io-console (0.5.6)
+    irb (1.2.4)
+      reline (>= 0.0.1)
+    king_konf (0.3.7)
+    minitest (5.14.2)
+    rake (13.0.1)
+    reline (0.1.4)
+      io-console (~> 0.5)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    ruby-kafka (0.7.2)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    ruby-kafka (1.2.0)
       digest-crc
-    simplecov (0.16.1)
+    simplecov (0.19.0)
       docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-    thor (0.20.0)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+    thor (1.0.1)
     thread_safe (0.3.6)
-    timecop (0.9.1)
-    tzinfo (1.2.5)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    waterdrop (1.2.4)
-      delivery_boy (~> 0.2)
-      dry-configurable (~> 0.7)
-      dry-monitor (~> 0.1)
-      dry-validation (~> 0.11)
-      null-logger (~> 0.1)
-      ruby-kafka (>= 0.6)
+    waterdrop (1.4.0)
+      delivery_boy (>= 0.2, < 2.x)
+      dry-configurable (~> 0.8)
+      dry-monitor (~> 0.3)
+      dry-validation (~> 1.2)
+      ruby-kafka (>= 0.7.8)
+      zeitwerk (~> 2.1)
+    zeitwerk (2.4.0)
 
 PLATFORMS
-  java
   ruby
 
 DEPENDENCIES
-  benchmark-ips
+  byebug
+  factory_bot
   karafka!
   rspec
   simplecov
-  timecop
 
 BUNDLED WITH
-   1.16.5
+   2.1.4

data/README.md

@@ -2,6 +2,12 @@
 
 [![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
 
+**Note**: Documentation presented here refers to Karafka `1.4`.
+
+If you are looking for the documentation for Karafka `1.3.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.3).
+
+## About Karafka
+
 Framework used to simplify Apache Kafka based Ruby applications development.
 
 Karafka allows you to capture everything that happens in your systems in large scale, providing you with a seamless and stable core for consuming and processing this data, without having to focus on things that are not your business domain.
@@ -27,15 +33,6 @@ Karafka based applications can be easily deployed to any type of infrastructure,
 * Docker
 * Terraform
 
-## Kafka 0.10 or prior
-
-If you're using Kafka 0.10, please lock `ruby-kafka` gem in your Gemfile to version `0.6.8`:
-
-```ruby
-gem 'karafka'
-gem 'ruby-kafka', '~> 0.6.8'
-```
-
 ## Support
 
 Karafka has a [Wiki pages](https://github.com/karafka/karafka/wiki) for almost everything and a pretty decent [FAQ](https://github.com/karafka/karafka/wiki/FAQ). It covers the whole installation, setup, and deployment along with other useful details on how to run Karafka.
@@ -52,7 +49,7 @@ If you're completely new to the subject, you can start with our "Kafka on Rails"
 If you want to get started with Kafka and Karafka as fast as possible, then the best idea is to just clone our example repository:
 
 ```bash
-git clone https://github.com/karafka/karafka-example-app ./example_app
+git clone https://github.com/karafka/example-app ./example_app
 ```
 
 then, just bundle install all the dependencies:
@@ -62,7 +59,7 @@ cd ./example_app
 bundle install
 ```
 
-and follow the instructions from the [example app Wiki](https://github.com/karafka/karafka-example-app/blob/master/README.md).
+and follow the instructions from the [example app Wiki](https://github.com/karafka/example-app/blob/master/README.md).
 
 **Note**: you need to ensure, that you have Kafka up and running and you need to configure Kafka seed_brokers in the ```karafka.rb``` file.
 
@@ -99,4 +96,4 @@ This project exists thanks to all the people who contribute.
 
 We are looking for sustainable sponsorship. If your company is relying on Karafka framework or simply want to see Karafka evolve faster to meet your requirements, please consider backing the project.
 
-Please contact [Maciej Mensfeld](mailto:maciej@coditsu.io) directly for more details.
+Please contact [Maciej Mensfeld](mailto:maciej@mensfeld.pl) directly for more details.

data/bin/karafka

@@ -10,7 +10,7 @@ else
   # However when it is unavailable, we still want to be able to run help command
   # and install command as they don't require configured app itself to run
   raise(
-    Karafka::Errors::MissingBootFile,
+    Karafka::Errors::MissingBootFileError,
     Karafka.boot_file
   ) unless %w[-h install].include?(ARGV[0])
 end

data/certs/mensfeld.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
+ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
+NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
+2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
+N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
+hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
+sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
+VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
+tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
+Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
+LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
+EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
+c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
+gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
+2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
+fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
+zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
+EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
+65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
+2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
+nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
+-----END CERTIFICATE-----

data/config/errors.yml

@@ -1,6 +1,39 @@
 en:
-  errors:
-    broker_schema?: >
-      has an invalid format.
-      Expected schema, host and port number.
-      Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
+  dry_validation:
+    errors:
+      invalid_broker_schema: >
+        has an invalid format
+        Expected schema, host and port number
+        Example: kafka://127.0.0.1:9092 or kafka+ssl://127.0.0.1:9092
+      invalid_certificate: >
+        is not a valid certificate
+      invalid_certificate_from_path: >
+        is not a valid certificate
+      invalid_private_key: >
+        is not a valid private key
+      max_timeout_size_for_exponential: >
+        pause_timeout cannot be more than pause_max_timeout
+      max_wait_time_limit:
+        max_wait_time cannot be more than socket_timeout
+      topics_names_not_unique: >
+        all topic names within a single consumer group must be unique
+      ssl_client_cert_with_ssl_client_cert_key: >
+        Both ssl_client_cert and ssl_client_cert_key need to be provided
+      ssl_client_cert_key_with_ssl_client_cert: >
+        Both ssl_client_cert_key and ssl_client_cert need to be provided
+      ssl_client_cert_chain_with_ssl_client_cert: >
+        Both ssl_client_cert_chain and ssl_client_cert need to be provided
+      ssl_client_cert_chain_with_ssl_client_cert_key: >
+        Both ssl_client_cert_chain and ssl_client_cert_key need to be provided
+      ssl_client_cert_key_password_with_ssl_client_cert_key: >
+        Both ssl_client_cert_key_password and ssl_client_cert_key need to be provided
+      does_not_respond_to_token: >
+        needs to respond to a #token method
+      required_usage_count: >
+        Given topic must be used at least once
+      pid_already_exists: >
+        Pidfile already exists
+      consumer_groups_inclusion: >
+        Unknown consumer group
+      does_not_exist:
+        Given file does not exist or cannot be read

data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    image: wurstmeister/kafka:1.0.1
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: localhost
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

data/karafka.gemspec

@@ -5,38 +5,39 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'karafka/version'
 
+# rubocop:disable Metrics/BlockLength
 Gem::Specification.new do |spec|
   spec.name        = 'karafka'
   spec.version     = ::Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld', 'Pavlo Vavruk', 'Adam Gwozdowski']
-  spec.email       = %w[maciej@coditsu.io pavlo.vavruk@gmail.com adam99g@gmail.com]
+  spec.email       = %w[maciej@mensfeld.pl pavlo.vavruk@gmail.com adam99g@gmail.com]
   spec.homepage    = 'https://github.com/karafka/karafka'
   spec.summary     = 'Ruby based framework for working with Apache Kafka'
   spec.description = 'Framework used to simplify Apache Kafka based Ruby applications development'
   spec.license     = 'MIT'
 
-  spec.add_dependency 'activesupport', '>= 4.0'
-  spec.add_dependency 'dry-configurable', '~> 0.7'
-  spec.add_dependency 'dry-inflector', '~> 0.1.1'
-  spec.add_dependency 'dry-monitor', '~> 0.1'
-  spec.add_dependency 'dry-validation', '~> 0.11'
-  spec.add_dependency 'envlogic', '~> 1.0'
-  spec.add_dependency 'multi_json', '>= 1.12'
+  spec.add_dependency 'dry-configurable', '~> 0.8'
+  spec.add_dependency 'dry-inflector', '~> 0.1'
+  spec.add_dependency 'dry-monitor', '~> 0.3'
+  spec.add_dependency 'dry-validation', '~> 1.2'
+  spec.add_dependency 'envlogic', '~> 1.1'
+  spec.add_dependency 'irb', '~> 1.0'
   spec.add_dependency 'rake', '>= 11.3'
-  spec.add_dependency 'require_all', '>= 1.4'
-  spec.add_dependency 'ruby-kafka', '>= 0.6'
-  spec.add_dependency 'thor', '~> 0.19'
-  spec.add_dependency 'waterdrop', '~> 1.2.4'
+  spec.add_dependency 'ruby-kafka', '>= 1.0.0'
+  spec.add_dependency 'thor', '>= 0.20'
+  spec.add_dependency 'waterdrop', '~> 1.4.0'
+  spec.add_dependency 'zeitwerk', '~> 2.1'
 
-  spec.post_install_message = <<~MSG
-    \e[93mWarning:\e[0m If you're using Kafka 0.10, please lock ruby-kafka in your Gemfile to version '0.6.8':
-    gem 'ruby-kafka', '~> 0.6.8'
-  MSG
+  spec.required_ruby_version = '>= 2.5.0'
 
-  spec.required_ruby_version = '>= 2.3.0'
+  if $PROGRAM_NAME.end_with?('gem')
+    spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
 
+  spec.cert_chain    = %w[certs/mensfeld.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]
 end
+# rubocop:enable Metrics/BlockLength

data/lib/karafka.rb

@@ -1,20 +1,22 @@
 # frozen_string_literal: true
 
 %w[
+  delegate
   English
   waterdrop
   kafka
   envlogic
+  json
   thor
+  forwardable
   fileutils
-  multi_json
-  require_all
   dry-configurable
   dry-validation
+  dry/events/publisher
   dry/inflector
   dry/monitor/notifications
-  active_support/callbacks
-  karafka/loader
+  dry/core/constants
+  zeitwerk
 ].each(&method(:require))
 
 # Karafka library
@@ -62,17 +64,9 @@ module Karafka
   end
 end
 
-%w[
-  callbacks
-  callbacks/*
-  setup/dsl
-  setup/config
-  status
-  schemas/config
-  schemas/consumer_group_topic
-  schemas/consumer_group
-].each { |path| require_all File.join(Karafka.core_root, path + '.rb') }
+Zeitwerk::Loader
+  .for_gem
+  .tap(&:setup)
+  .tap(&:eager_load)
 
-Karafka::Loader.load!(Karafka.core_root)
 Kafka::Consumer.prepend(Karafka::Patches::RubyKafka)
-Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)