karafka 1.4.0 → 2.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45788f5dd509d8a61a1480ff5c1b41f89b683d2100e0dcedaf6e35e6ce3af447
-  data.tar.gz: d5e5638fd2a83b91f78cf1f38d03bd2594fb562a0cc8ad6cf2e45f60e72cc692
+  metadata.gz: f78f7cb985880d9172961be96386a0ebd37735831915f0cb9b9b46c832d2e9a9
+  data.tar.gz: a7f5a27cb3a6f0fa5185f32e5df7615f8a6013ef0018c85fb16c87de346f9362
 SHA512:
-  metadata.gz: 75b2863022df6dc78f23ca557450a9757e137d110f9a6c49a29f572cd701863143e638394c341e8f982bdce4255df923879c8410ce081503c43e3f9a0ce2e777
-  data.tar.gz: ec0eee525c34634d15b04c85a492e329d4749f179ff2c21d3ce2e50cccb4b6d061465c13df322a026b8924974f743277448464db818e13d73ef90f5ce930f796
+  metadata.gz: 5b6fed517f69a2bd84b16e82d266251dd4b7564d52f466745f3a93be9106f866ec658581c38b1a0913dd9e126a17cf4b1f5e85c888447c6e1e4b790453e7c87e
+  data.tar.gz: 25cfa154028dd12519d7d4b72606242a4058c9fabd1655ba727a9d73a905a942cd80374e2f7357f410a098b74b623966fa840b1c6725f71d630bf3ce92db187e

data/.github/workflows/ci.yml

@@ -1,52 +1,123 @@
 name: ci
 
+concurrency: ci-${{ github.ref }}
+
 on:
+  pull_request:
   push:
   schedule:
     - cron:  '0 1 * * *'
 
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
 jobs:
+  diffend:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.1
+          bundler-cache: true
+
+      - name: Install Diffend plugin
+        run: bundle plugin install diffend
+
+      - name: Bundle Secure
+        run: bundle secure
+
+  coditsu:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run Coditsu
+        run: \curl -sSL https://api.coditsu.io/run/ci | bash
+
   specs:
     runs-on: ubuntu-latest
+    needs: diffend
     strategy:
       fail-fast: false
       matrix:
         ruby:
+          - '3.1'
+          - '3.0'
           - '2.7'
-          - '2.6'
-          - '2.5'
         include:
-          - ruby: '2.7'
+          - ruby: '3.1'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v2
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+
+      - name: Start Kafka with docker-compose
+        run: |
+          docker-compose up -d
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
-      - name: Install latest bundler
-        run: |
-          gem install bundler --no-document
-          bundle config set without 'tools benchmarks docs'
-      - name: Bundle install
-        run: |
-          bundle config set without development
-          bundle install --jobs 4 --retry 3
-      - name: Run Kafka with docker-compose
-        run: docker-compose up -d
-      - name: Run all tests
+          bundler-cache: true
+
+      - name: Run all specs
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bundle exec rspec
-  coditsu:
+
+  integrations:
     runs-on: ubuntu-latest
+    needs: diffend
     strategy:
       fail-fast: false
+      matrix:
+        ruby:
+          - '3.1'
+          - '3.0'
+          - '2.7'
+        include:
+          - ruby: '3.1'
+            coverage: 'true'
     steps:
       - uses: actions/checkout@v2
+      - name: Install package dependencies
+        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+
+      - name: Start Kafka with docker-compose
+        run: |
+          docker-compose up -d
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
         with:
-          fetch-depth: 0
-      - name: Run Coditsu
-        run: \curl -sSL https://api.coditsu.io/run/ci | bash
+          ruby-version: ${{matrix.ruby}}
+
+      - name: Install latest Bundler
+        run: |
+          gem install bundler --no-document
+          gem update --system --no-document
+          bundle config set without 'tools benchmarks docs'
+
+      - name: Bundle install
+        run: |
+          bundle config set without development
+          bundle install
+
+      - name: Run integration tests
+        env:
+          KARAFKA_PRO_LICENSE_TOKEN: ${{ secrets.KARAFKA_PRO_LICENSE_TOKEN }}
+          GITHUB_COVERAGE: ${{matrix.coverage}}
+        run: bin/integrations

data/.ruby-version

@@ -1 +1 @@
-2.7.1
+3.1.2

data/CHANGELOG.md

@@ -1,8 +1,372 @@
 # Karafka framework changelog
 
+## 2.0.10 (2022-09-23)
+- Improve error recovery by delegating the recovery to the existing `librdkafka` instance.
+
+## 2.0.9 (2022-09-22)
+- Fix Singleton not visible when used in PORO (#1034)
+- Divide pristine specs into pristine and poro. Pristine will still have helpers loaded, poro will have nothing.
+- Fix a case where `manual_offset_management` offset upon error is not reverted to the first message in a case where there were no markings as consumed at all for multiple batches.
+- Implement small reliability improvements around marking as consumed.
+- Introduce a config sanity check to make sure Virtual Partitions are not used with manual offset management.
+- Fix a possibility of using `active_job_topic` with Virtual Partitions and manual offset management (ActiveJob still can use due to atomicity of jobs).
+- Move seek offset ownership to the coordinator to allow Virtual Partitions further development.
+- Improve client shutdown in specs.
+- Do not reset client on network issue and rely on `librdkafka` to do so.
+- Allow for nameless (anonymous) subscription groups (#1033)
+
+## 2.0.8 (2022-09-19)
+- [Breaking change] Rename Virtual Partitions `concurrency` to `max_partitions` to avoid confusion  (#1023).
+-  Allow for block based subscription groups management (#1030).
+
+## 2.0.7 (2022-09-05)
+- [Breaking change] Redefine the Virtual Partitions routing DSL to accept concurrency
+- Allow for `concurrency` setting in Virtual Partitions to extend or limit number of jobs per regular partition. This allows to make sure, we do not use all the threads on virtual partitions jobs
+- Allow for creation of as many Virtual Partitions as needed, without taking global `concurrency` into consideration
+
+## 2.0.6 (2022-09-02)
+- Improve client closing.
+- Fix for: Multiple LRJ topics fetched concurrently block ability for LRJ to kick in (#1002)
+- Introduce a pre-enqueue sync execution layer to prevent starvation cases for LRJ
+- Close admin upon critical errors to prevent segmentation faults
+- Add support for manual subscription group management (#852)
+
+## 2.0.5 (2022-08-23)
+- Fix unnecessary double new line in the `karafka.rb` template for Ruby on Rails
+- Fix a case where a manually paused partition would not be processed after rebalance (#988)
+- Increase specs stability.
+- Lower concurrency of execution of specs in Github CI.
+
+## 2.0.4 (2022-08-19)
+- Fix hanging topic creation (#964)
+- Fix conflict with other Rails loading libraries like `gruf` (#974)
+
+## 2.0.3 (2022-08-09)
+- Update boot info on server startup.
+- Update `karafka info` with more descriptive Ruby version info.
+- Fix issue where when used with Rails in development, log would be too verbose.
+- Fix issue where Zeitwerk with Rails would not load Pro components despite license being present.
+
+## 2.0.2 (2022-08-07)
+- Bypass issue with Rails reload in development by releasing the connection (https://github.com/rails/rails/issues/44183).
+
+## 2.0.1 (2022-08-06)
+- Provide `Karafka::Admin` for creation and destruction of topics and fetching cluster info.
+- Update integration specs to always use one-time disposable topics.
+- Remove no longer needed `wait_for_kafka` script.
+- Add more integration specs for cover offset management upon errors.
+
+## 2.0.0 (2022-08-05)
+
+This changelog describes changes between `1.4` and `2.0`. Please refer to appropriate release notes for changes between particular `rc` releases.
+
+Karafka 2.0 is a **major** rewrite that brings many new things to the table but also removes specific concepts that happened not to be as good as I initially thought when I created them.
+
+Please consider getting a Pro version if you want to **support** my work on the Karafka ecosystem!
+
+For anyone worried that I will start converting regular features into Pro: This will **not** happen. Anything free and fully OSS in Karafka 1.4 will **forever** remain free. Most additions and improvements to the ecosystem are to its free parts. Any feature that is introduced as a free and open one will not become paid.
+
+### Additions
+
+This section describes **new** things and concepts introduced with Karafka 2.0.
+
+Karafka 2.0:
+
+- Introduces multi-threaded support for [concurrent work](https://github.com/karafka/karafka/wiki/Concurrency-and-multithreading) consumption for separate partitions as well as for single partition work via [Virtual Partitions](https://github.com/karafka/karafka/wiki/Pro-Virtual-Partitions).
+- Introduces [Active Job adapter](https://github.com/karafka/karafka/wiki/Active-Job) for using Karafka as a jobs backend with Ruby on Rails Active Job.
+- Introduces fully automatic integration end-to-end [test suite](https://github.com/karafka/karafka/tree/master/spec/integrations) that checks any case I could imagine.
+- Introduces [Virtual Partitions](https://github.com/karafka/karafka/wiki/Pro-Virtual-Partitions) for ability to parallelize work of a single partition.
+- Introduces [Long-Running Jobs](https://github.com/karafka/karafka/wiki/Pro-Long-Running-Jobs) to allow for work that would otherwise exceed the `max.poll.interval.ms`.
+- Introduces the [Enhanced Scheduler](https://github.com/karafka/karafka/wiki/Pro-Enhanced-Scheduler) that uses a non-preemptive LJF (Longest Job First) algorithm instead of a a FIFO (First-In, First-Out) one.
+- Introduces [Enhanced Active Job adapter](https://github.com/karafka/karafka/wiki/Pro-Enhanced-Active-Job) that is optimized and allows for strong ordering of jobs and more.
+- Introduces seamless [Ruby on Rails integration](https://github.com/karafka/karafka/wiki/Integrating-with-Ruby-on-Rails-and-other-frameworks) via `Rails::Railte` without need for any extra configuration.
+- Provides `#revoked` [method](https://github.com/karafka/karafka/wiki/Consuming-messages#shutdown-and-partition-revocation-handlers) for taking actions upon topic revocation.
+- Emits underlying async errors emitted from `librdkafka` via the standardized `error.occurred` [monitor channel](https://github.com/karafka/karafka/wiki/Error-handling-and-back-off-policy#error-tracking).
+- Replaces `ruby-kafka` with `librdkafka` as an underlying driver.
+- Introduces official [EOL policies](https://github.com/karafka/karafka/wiki/Versions-Lifecycle-and-EOL).
+- Introduces [benchmarks](https://github.com/karafka/karafka/tree/master/spec/benchmarks) that can be used to profile Karafka.
+- Introduces a requirement that the end user code **needs** to be [thread-safe](https://github.com/karafka/karafka/wiki/FAQ#does-karafka-require-gems-to-be-thread-safe).
+- Introduces a [Pro subscription](https://github.com/karafka/karafka/wiki/Build-vs-Buy) with a [commercial license](https://github.com/karafka/karafka/blob/master/LICENSE-COMM) to fund further ecosystem development.
+
+### Deletions
+
+This section describes things that are **no longer** part of the Karafka ecosystem.
+
+Karafka 2.0:
+
+- Removes topics mappers concept completely.
+- Removes pidfiles support.
+- Removes daemonization support.
+- Removes support for using `sidekiq-backend` due to introduction of [multi-threading](https://github.com/karafka/karafka/wiki/Concurrency-and-multithreading).
+- Removes the `Responders` concept in favour of WaterDrop producer usage.
+- Removes completely all the callbacks in favour of finalizer method `#shutdown`.
+- Removes single message consumption mode in favour of [documentation](https://github.com/karafka/karafka/wiki/Consuming-messages#one-at-a-time) on how to do it easily by yourself.
+
+### Changes
+
+This section describes things that were **changed** in Karafka but are still present.
+
+Karafka 2.0:
+
+- Uses only instrumentation that comes from Karafka. This applies also to notifications coming natively from `librdkafka`. They are now piped through Karafka prior to being dispatched.
+- Integrates WaterDrop `2.x` tightly with autoconfiguration inheritance and an option to redefine it.
+- Integrates with the `karafka-testing` gem for RSpec that also has been updated.
+- Updates `cli info` to reflect the `2.0` details.
+- Stops validating `kafka` configuration beyond minimum as the rest is handled by `librdkafka`.
+- No longer uses `dry-validation`.
+- No longer uses `dry-monitor`.
+- No longer uses `dry-configurable`.
+- Lowers general external dependencies three **heavily**.
+- Renames `Karafka::Params::BatchMetadata` to `Karafka::Messages::BatchMetadata`.
+- Renames `Karafka::Params::Params` to `Karafka::Messages::Message`.
+- Renames `#params_batch` in consumers to `#messages`.
+- Renames `Karafka::Params::Metadata` to `Karafka::Messages::Metadata`.
+- Renames `Karafka::Fetcher` to `Karafka::Runner` and align notifications key names.
+- Renames `StdoutListener` to `LoggerListener`.
+- Reorganizes [monitoring and logging](https://github.com/karafka/karafka/wiki/Monitoring-and-logging) to match new concepts.
+- Notifies on fatal worker processing errors.
+- Contains updated install templates for Rails and no-non Rails.
+- Changes how the routing style (`0.5`) behaves. It now builds a single consumer group instead of one per topic.
+- Introduces changes that will allow me to build full web-UI in the upcoming `2.1`.
+- Contains updated example apps.
+- Standardizes error hooks for all error reporting (`error.occurred`).
+- Changes license to `LGPL-3.0`.
+- Introduces a `karafka-core` dependency that contains common code used across the ecosystem.
+- Contains updated [wiki](https://github.com/karafka/karafka/wiki) on everything I could think of.
+
+### What's ahead
+
+Karafka 2.0 is just the beginning.
+
+There are several things in the plan already for 2.1 and beyond, including a web dashboard, at-rest encryption, transactions support, and more.
+
+## 2.0.0.rc6 (2022-08-05)
+- Update licenser to use a gem based approach based on `karafka-license`.
+- Do not mark intermediate jobs as consumed when Karafka runs Enhanced Active Job with Virtual Partitions.
+- Improve development experience by adding fast cluster state changes refresh (#944)
+- Improve the license loading.
+
+## 2.0.0.rc5 (2022-08-01)
+- Improve specs stability
+- Improve forceful shutdown
+- Add support for debug `TTIN` backtrace printing
+- Fix a case where logger listener would not intercept `warn` level
+- Require `rdkafka` >= `0.12`
+- Replace statistics decorator with the one from `karafka-core`
+
+## 2.0.0.rc4 (2022-07-28)
+- Remove `dry-monitor`
+- Use `karafka-core`
+- Improve forceful shutdown resources finalization
+- Cache consumer client name
+
+## 2.0.0.rc3 (2022-07-26)
+- Fix Pro partitioner hash function may not utilize all the threads (#907).
+- Improve virtual partitions messages distribution.
+- Add StatsD/DataDog optional monitoring listener + dashboard template.
+- Validate that Pro consumer is always used for Pro subscription.
+- Improve ActiveJob consumer shutdown behaviour.
+- Change default `max_wait_time` to 1 second.
+- Change default `max_messages` to 100 (#915).
+- Move logger listener polling reporting level to debug when no messages (#916).
+- Improve stability on aggressive rebalancing (multiple rebalances in a short period).
+- Improve specs stability.
+- Allow using `:key` and `:partition_key` for Enhanced Active Job partitioning.
+
+## 2.0.0.rc2 (2022-07-19)
+- Fix `example_consumer.rb.erb` `#shutdown` and `#revoked` signatures to correct once.
+- Improve the install user experience (print status and created files).
+- Change default `max_wait_time` from 10s to 5s.
+- Remove direct dependency on `dry-configurable` in favour of a home-brew.
+- Remove direct dependency on `dry-validation` in favour of a home-brew.
+
+## 2.0.0-rc1 (2022-07-08)
+- Extract consumption partitioner out of listener inline code.
+- Introduce virtual partitioner concept for parallel processing of data from a single topic partition.
+- Improve stability when there kafka internal errors occur while polling.
+- Fix a case where we would resume a LRJ partition upon rebalance where we would reclaim the partition while job was still running.
+- Do not revoke pauses for lost partitions. This will allow to un-pause reclaimed partitions when LRJ jobs are done.
+- Fail integrations by default (unless configured otherwise) if any errors occur during Karafka server execution.
+
+## 2.0.0-beta5 (2022-07-05)
+- Always resume processing of a revoked partition upon assignment.
+- Improve specs stability.
+- Fix a case where revocation job would be executed on partition for which we never did any work.
+- Introduce a jobs group coordinator for easier jobs management.
+- Improve stability of resuming paused partitions that were revoked and re-assigned.
+- Optimize reaction time on partition ownership changes.
+- Fix a bug where despite setting long max wait time, we would return messages prior to it while not reaching the desired max messages count.
+- Add more integration specs related to polling limits.
+- Remove auto-detection of re-assigned partitions upon rebalance as for too fast rebalances it could not be accurate enough. It would also mess up in case of rebalances that would happen right after a `#seek` was issued for a partition.
+- Optimize the removal of pre-buffered lost partitions data.
+- Always run `#revoked` when rebalance with revocation happens.
+- Evict executors upon rebalance, to prevent race-conditions.
+- Align topics names for integration specs.
+
+## 2.0.0-beta4 (2022-06-20)
+- Rename job internal api methods from `#prepare` to `#before_call` and from `#teardown` to `#after_call` to abstract away jobs execution from any type of executors and consumers logic
+- Remove ability of running `before_consume` and `after_consume` completely. Those should be for internal usage only.
+- Reorganize how Pro consumer and Pro AJ consumers inherit.
+- Require WaterDrop `2.3.1`.
+- Add more integration specs for rebalancing and max poll exceeded.
+- Move `revoked?` state from PRO to regular Karafka.
+- Use return value of `mark_as_consumed!` and `mark_as_consumed` as indicator of partition ownership + use it to switch the ownership state.
+- Do not remove rebalance manager upon client reset and recovery. This will allow us to keep the notion of lost partitions, so we can run revocation jobs for blocking jobs that exceeded the max poll interval.
+- Run revocation jobs upon reaching max poll interval for blocking jobs.
+- Early exit `poll` operation upon partition lost or max poll exceeded event.
+- Always reset consumer instances on timeout exceeded.
+- Wait for Kafka to create all the needed topics before running specs in CI.
+
+## 2.0.0-beta3 (2022-06-14)
+- Jobs building responsibility extracted out of the listener code base.
+- Fix a case where specs supervisor would try to kill no longer running process (#868)
+- Fix an instable integration spec that could misbehave under load
+- Commit offsets prior to pausing partitions to ensure that the latest offset is always committed
+- Fix a case where consecutive CTRL+C (non-stop) would case an exception during forced shutdown
+- Add missing `consumer.prepared.error` into `LoggerListener`
+- Delegate partition resuming from the consumers to listeners threads.
+- Add support for Long-Running Jobs (LRJ) for ActiveJob [PRO]
+- Add support for Long-Running Jobs for consumers [PRO]
+- Allow `active_job_topic` to accept a block for extra topic related settings
+- Remove no longer needed logger threads
+- Auto-adapt number of processes for integration specs based on the number of CPUs
+- Introduce an integration spec runner that prints everything to stdout (better for development)
+- Introduce extra integration specs for various ActiveJob usage scenarios
+- Rename consumer method `#prepared` to `#prepare` to reflect better its use-case
+- For test and dev raise an error when expired license key is used (never for non dev)
+- Add worker related monitor events (`worker.process` and `worker.processed`)
+- Update `LoggerListener` to include more useful information about processing and polling messages
+
+## 2.0.0-beta2 (2022-06-07)
+- Abstract away notion of topics groups (until now it was just an array)
+- Optimize how jobs queue is closed. Since we enqueue jobs only from the listeners, we can safely close jobs queue once listeners are done. By extracting this responsibility from listeners, we remove corner cases and race conditions. Note here: for non-blocking jobs we do wait for them to finish while running the `poll`. This ensures, that for async jobs that are long-living, we do not reach `max.poll.interval`.
+- `Shutdown` jobs are executed in workers to align all the jobs behaviours.
+- `Shutdown` jobs are always blocking.
+- Notion of `ListenersBatch` was introduced similar to `WorkersBatch` to abstract this concept.
+- Change default `shutdown_timeout` to be more than `max_wait_time` not to cause forced shutdown when no messages are being received from Kafka.
+- Abstract away scheduling of revocation and shutdown jobs for both default and pro schedulers
+- Introduce a second (internal) messages buffer to distinguish between raw messages buffer and karafka messages buffer
+- Move messages and their metadata remap process to the listener thread to allow for their inline usage
+- Change how we wait in the shutdown phase, so shutdown jobs can still use Kafka connection even if they run for a longer period of time. This will prevent us from being kicked out from the group early.
+- Introduce validation that ensures, that `shutdown_timeout` is more than `max_wait_time`. This will prevent users from ending up with a config that could lead to frequent forceful shutdowns.
+
+## 2.0.0-beta1 (2022-05-22)
+- Update the jobs queue blocking engine and allow for non-blocking jobs execution
+- Provide `#prepared` hook that always runs before the fetching loop is unblocked
+- [Pro] Introduce performance tracker for scheduling optimizer
+- Provide ability to pause (`#pause`) and resume (`#resume`) given partitions from the consumers
+- Small integration specs refactoring + specs for pausing scenarios
+
+## 2.0.0-alpha6 (2022-04-17)
+- Fix a bug, where upon missing boot file and Rails, railtie would fail with a generic exception (#818) 
+- Fix an issue with parallel pristine specs colliding with each other during `bundle install` (#820)
+- Replace `consumer.consume` with `consumer.consumed` event to match the behaviour
+- Make sure, that offset committing happens before the `consumer.consumed` event is propagated
+- Fix for failing when not installed (just a dependency) (#817)
+- Evict messages from partitions that were lost upon rebalancing (#825)
+- Do **not** run `#revoked` on partitions that were lost and assigned back upon rebalancing (#825)
+- Remove potential duplicated that could occur upon rebalance with re-assigned partitions (#825)
+- Optimize integration test suite additional consumers shutdown process (#828)
+- Optimize messages eviction and duplicates removal on poll stopped due to lack of messages
+- Add static group membership integration spec
+
+## 2.0.0-alpha5 (2022-04-03)
+- Rename StdoutListener to LoggerListener (#811)
+
+## 2.0.0-alpha4 (2022-03-20)
+- Rails support without ActiveJob queue adapter usage (#805)
+
+## 2.0.0-alpha3 (2022-03-16)
+- Restore 'app.initialized' state and add notification on it
+- Fix the installation flow for Rails and add integration tests for this scenario
+- Add more integration tests covering some edge cases
+
+## 2.0.0-alpha2 (2022-02-19)
+- Require `kafka` keys to be symbols
+- [Pro] Added ActiveJob Pro adapter
+- Small updates to the license and docs
+
+## 2.0.0-alpha1 (2022-01-30)
+- Change license to `LGPL-3.0`
+- [Pro] Introduce a Pro subscription
+- Switch from `ruby-kafka` to `librdkafka` as an underlying driver
+- Introduce fully automatic integration tests that go through the whole server lifecycle
+- Integrate WaterDrop tightly with autoconfiguration inheritance and an option to redefine it
+- Multi-threaded support for concurrent jobs consumption (when in separate topics and/or partitions)
+- Introduce subscriptions groups concept for better resources management
+- Remove completely all the callbacks in favour of finalizer method `#on_shutdown`
+- Provide `on_revoked` method for taking actions upon topic revoke
+- Remove single message consumption mode in favour of documentation on how to do it easily by yourself
+- Provide sync and async offset management with async preferred
+- Introduce seamless Ruby on Rails integration via `Rails::Railte`
+- Update `cli info` to reflect the `2.0` details
+- Remove responders in favour of WaterDrop `2.0` producer
+- Remove pidfiles support
+- Remove daemonization support
+- Stop validating `kafka` configuration beyond minimum as it is handled by `librdkafka`
+- Remove topics mappers concept
+- Reorganize monitoring to match new concepts
+- Notify on fatal worker processing errors
+- Rename `Karafka::Params::BatchMetadata` to `Karafka::Messages::BatchMetadata`
+- Rename `Karafka::Params::Params` to `Karafka::Messages::Message`
+- Rename `#params_batch` in consumers to `#messages`
+- Rename `Karafka::Params::Metadata` to `Karafka::Messages::Metadata`
+- Allow for processing work of multiple consumer groups by the same worker poll
+- Rename `Karafka::Fetcher` to `Karafka::Runner` and align notifications key names
+- Update install templates
+- `sidekiq-backend` is no longer supported
+- `testing` gem for RSpec has been updated
+- `WaterDrop` `2.1+` support
+- Simple routing style (`0.5`) now builds a single consumer group instead of one per topic
+- Example apps were updated
+- Hook for underlying statistics emitted from librdkafka have been added
+- Hook for underlying async errors emitted from  librdkafka have been added
+- ActiveJob Rails adapter
+- Added benchmarks that can be used to profile Karafka
+- Standardize error hook for all error reporting
+
+## 1.4.11 (2021-12-04)
+- Source code metadata url added to the gemspec
+- Gem bump
+
+## 1.4.10 (2021-10-30)
+- update gems requirements in the gemspec (nijikon)
+
+## 1.4.9 (2021-09-29)
+- fix `dry-configurable` deprecation warnings for default value as positional argument
+
+## 1.4.8 (2021-09-08)
+- Allow 'rails' in Gemfile to enable rails-aware generator (rewritten)
+
+## 1.4.7 (2021-09-04)
+- Update ruby-kafka to `1.4.0`
+- Support for `resolve_seed_brokers` option (with Azdaroth)
+- Set minimum `ruby-kafka` requirement to `1.3.0`
+
+## 1.4.6 (2021-08-05)
+- #700 Fix Ruby 3 compatibility issues in Connection::Client#pause (MmKolodziej)
+
+## 1.4.5 (2021-06-16)
+- Fixup logger checks for non-writeable logfile (ojab)
+- #689 - Update the stdout initialization message for framework initialization
+
+## 1.4.4 (2021-04-19)
+- Remove Ruby 2.5 support and update minimum Ruby requirement to 2.6
+- Remove rake dependency
+
+## 1.4.3 (2021-03-24)
+- Fixes for Ruby 3.0 compatibility
+
+## 1.4.2 (2021-02-16)
+- Rescue Errno::EROFS in ensure_dir_exists (unasuke)
+
+## 1.4.1 (2020-12-04)
+- Return non-zero exit code when printing usage
+- Add support for :assignment_strategy for consumers
+
 ## 1.4.0 (2020-09-05)
 - Rename `Karafka::Params::Metadata` to `Karafka::Params::BatchMetadata`
-` Rename consumer `#metadata` to `#batch_metadata`
+- Rename consumer `#metadata` to `#batch_metadata`
 - Separate metadata (including Karafka native metadata) from the root of params (backwards compatibility preserved thanks to rabotyaga)
 - Remove metadata hash dependency
 - Remove params dependency on a hash in favour of PORO

data/CONTRIBUTING.md

@@ -1,41 +1,32 @@
-# Contribute
+# Contributing
 
 ## Introduction
 
-First, thank you for considering contributing to karafka! It's people like you that make the open source community such a great community! 😊
+First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community! 😊
 
 We welcome any type of contribution, not only code. You can help with:
 - **QA**: file bug reports, the more details you can give the better (e.g. screenshots with the console open)
 - **Marketing**: writing blog posts, howto's, printing stickers, ...
 - **Community**: presenting the project at meetups, organizing a dedicated meetup for the local community, ...
-- **Code**: take a look at the [open issues](issues). Even if you can't write code, commenting on them, showing that you care about a given issue matters. It helps us triage them.
-
-## Your First Contribution
-
-Working on your first Pull Request? You can learn how from this *free* series, [How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github).
+- **Code**: take a look at the [open issues](https://github.com/karafka/karafka/issues). Even if you can't write code, commenting on them, showing that you care about a given issue matters. It helps us triage them.
 
 ## Submitting code
 
 Any code change should be submitted as a pull request. The description should explain what the code does and give steps to execute it. The pull request should also contain tests.
 
-## Code review process
+### Code review process
 
 Each pull request must pass all the rspec specs and meet our quality requirements.
 
 To check if everything is as it should be, we use [Coditsu](https://coditsu.io) that combines multiple linters and code analyzers for both code and documentation. Once you're done with your changes, submit a pull request.
 
-Coditsu will automatically check your work against our quality standards. You can find your commit check results on the [builds page](https://app.coditsu.io/karafka/commit_builds) of Karafka organization.
-
-[![coditsu](https://coditsu.io/assets/quality_bar.svg)](https://app.coditsu.io/karafka/commit_builds)
+### Contributing to Pro components
 
-## Questions
-
-If you have any questions, create an [issue](issue) (protip: do a quick search first to see if someone else didn't ask the same question before!).
-You can also reach us at hello@karafka.opencollective.com.
+All of Karafka components are open-source. However, the `Pro` components are licenses under `LICENSE-COMM`.
 
-## Credits
+By sending a pull request to the pro components, you are agreeing to transfer the copyright of your code to Maciej Mensfeld.
 
-### Contributors
+## Questions
 
-Thank you to all the people who have already contributed to karafka!
-<a href="graphs/contributors"><img src="https://opencollective.com/karafka/contributors.svg?width=890" /></a>
+If you have any questions, create an [issue](https://github.com/karafka/karafka/issues) (protip: do a quick search first to see if someone else didn't ask the same question before!).
+You can also reach us at contact@karafka.io.

data/Gemfile

@@ -6,6 +6,12 @@ plugin 'diffend'
 
 gemspec
 
+# Karafka gem does not require this but we add it here so we can test the integration with
+# ActiveJob much easier
+group :integrations do
+  gem 'activejob'
+end
+
 group :test do
   gem 'byebug'
   gem 'factory_bot'