karafka 2.5.9 → 2.6.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +34 -0
- data/certs/expired.txt +83 -0
- data/config/locales/errors.yml +17 -7
- data/karafka.gemspec +3 -3
- data/lib/active_job/queue_adapters/karafka_adapter.rb +1 -2
- data/lib/karafka/active_job/job_extensions.rb +1 -2
- data/lib/karafka/admin/configs/resource.rb +1 -2
- data/lib/karafka/admin/consumer_groups.rb +109 -98
- data/lib/karafka/admin/isolation_levels.rb +22 -0
- data/lib/karafka/admin/topics.rb +103 -8
- data/lib/karafka/admin.rb +59 -31
- data/lib/karafka/app.rb +16 -5
- data/lib/karafka/base_consumer.rb +2 -2
- data/lib/karafka/cli/contracts/server.rb +4 -4
- data/lib/karafka/cli/info.rb +1 -1
- data/lib/karafka/cli/topics/base.rb +10 -18
- data/lib/karafka/cli/topics/repartition.rb +1 -1
- data/lib/karafka/connection/client.rb +40 -9
- data/lib/karafka/connection/consumer_groups/rebalance_manager.rb +120 -0
- data/lib/karafka/connection/listener.rb +8 -7
- data/lib/karafka/connection/listeners_batch.rb +1 -1
- data/lib/karafka/connection/mode.rb +1 -2
- data/lib/karafka/connection/raw_messages_buffer.rb +0 -5
- data/lib/karafka/declaratives/builder.rb +65 -0
- data/lib/karafka/declaratives/contracts/topic.rb +28 -0
- data/lib/karafka/declaratives/repository.rb +52 -0
- data/lib/karafka/declaratives/topic.rb +100 -0
- data/lib/karafka/declaratives.rb +9 -0
- data/lib/karafka/helpers/interval_runner.rb +2 -2
- data/lib/karafka/instrumentation/assignments_tracker.rb +65 -2
- data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb +56 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/rebalance.rb +93 -0
- data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb +59 -0
- data/lib/karafka/instrumentation/logger_listener.rb +27 -9
- data/lib/karafka/instrumentation/notifications.rb +2 -0
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +14 -17
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +8 -9
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +7 -3
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +13 -10
- data/lib/karafka/licenser.rb +16 -3
- data/lib/karafka/pro/active_job/consumer.rb +1 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +1 -2
- data/lib/karafka/pro/admin/recovery.rb +19 -19
- data/lib/karafka/pro/base_consumer.rb +3 -3
- data/lib/karafka/pro/cli/contracts/server.rb +5 -5
- data/lib/karafka/pro/cli/parallel_segments/base.rb +7 -7
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +4 -4
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +6 -6
- data/lib/karafka/pro/iterator/tpl_builder.rb +38 -18
- data/lib/karafka/pro/loader.rb +15 -12
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/consumer.rb +84 -0
- data/lib/karafka/pro/processing/consumer_groups/adaptive_iterator/tracker.rb +97 -0
- data/lib/karafka/pro/processing/consumer_groups/collapser.rb +84 -0
- data/lib/karafka/pro/processing/consumer_groups/coordinator.rb +202 -0
- data/lib/karafka/pro/processing/consumer_groups/coordinators/errors_tracker.rb +124 -0
- data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb +157 -0
- data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb +212 -0
- data/lib/karafka/pro/processing/{filters/expirer.rb → consumer_groups/executor.rb} +17 -31
- data/lib/karafka/pro/processing/{jobs/periodic.rb → consumer_groups/expansions_selector.rb} +18 -21
- data/lib/karafka/pro/processing/consumer_groups/filters/base.rb +103 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb +92 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb +78 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb +99 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb +106 -0
- data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb +79 -0
- data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/consume_non_blocking.rb +21 -17
- data/lib/karafka/pro/processing/{virtual_partitions/distributors/consistent.rb → consumer_groups/jobs/eofed_non_blocking.rb} +16 -14
- data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb +64 -0
- data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/periodic_non_blocking.rb +16 -11
- data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/revoked_non_blocking.rb +19 -15
- data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb +95 -0
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb +66 -0
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb +154 -0
- data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb +68 -0
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb +102 -0
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb +115 -0
- data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb +96 -0
- data/lib/karafka/pro/processing/consumer_groups/partitioner.rb +98 -0
- data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb +90 -0
- data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb +154 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb +93 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb +99 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom.rb +92 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_mom_vp.rb +90 -0
- data/lib/karafka/pro/processing/{strategies/aj/dlq_ftr_lrj_mom.rb → consumer_groups/strategies/aj/dlq_lrj_mom.rb} +37 -39
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_lrj_mom_vp.rb +90 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom.rb +84 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_mom_vp.rb +89 -0
- data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/aj/ftr_lrj_mom.rb +20 -15
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_lrj_mom_vp.rb +91 -0
- data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/aj/ftr_mom.rb +20 -15
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/ftr_mom_vp.rb +80 -0
- data/lib/karafka/pro/processing/{strategies/mom/default.rb → consumer_groups/strategies/aj/lrj_mom.rb} +18 -22
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/lrj_mom_vp.rb +106 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom.rb +58 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/aj/mom_vp.rb +74 -0
- data/lib/karafka/pro/processing/{strategies/lrj/vp.rb → consumer_groups/strategies/base.rb} +9 -14
- data/lib/karafka/pro/processing/consumer_groups/strategies/default.rb +421 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/default.rb +285 -0
- data/lib/karafka/pro/processing/{strategies/dlq/lrj.rb → consumer_groups/strategies/dlq/ftr.rb} +30 -29
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj.rb +95 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_mom.rb +97 -0
- data/lib/karafka/pro/processing/{executor.rb → consumer_groups/strategies/dlq/ftr_lrj_mom_vp.rb} +26 -15
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_lrj_vp.rb +63 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom.rb +97 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/ftr_mom_vp.rb +63 -0
- data/lib/karafka/pro/{routing/features/patterns/patterns.rb → processing/consumer_groups/strategies/dlq/ftr_vp.rb} +22 -12
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj.rb +83 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_mom.rb +100 -0
- data/lib/karafka/pro/processing/{strategies → consumer_groups/strategies}/dlq/lrj_mom_vp.rb +18 -13
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/lrj_vp.rb +61 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/mom.rb +96 -0
- data/lib/karafka/pro/processing/{strategies/lrj → consumer_groups/strategies/dlq}/mom_vp.rb +19 -15
- data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb +62 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb +146 -0
- data/lib/karafka/pro/processing/{strategies/mom/ftr.rb → consumer_groups/strategies/ftr/vp.rb} +20 -28
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb +119 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb +94 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb +92 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb +62 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb +61 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb +101 -0
- data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb +60 -0
- data/lib/karafka/pro/processing/{strategies/mom → consumer_groups/strategies/lrj}/vp.rb +18 -12
- data/lib/karafka/pro/processing/{strategies/aj/mom_vp.rb → consumer_groups/strategies/mom/default.rb} +22 -23
- data/lib/karafka/pro/processing/{strategies/aj/ftr_mom_vp.rb → consumer_groups/strategies/mom/ftr.rb} +28 -28
- data/lib/karafka/pro/processing/{virtual_partitions/distributors/base.rb → consumer_groups/strategies/mom/ftr_vp.rb} +19 -14
- data/lib/karafka/pro/processing/{strategies/aj/mom.rb → consumer_groups/strategies/mom/vp.rb} +16 -12
- data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb +197 -0
- data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb +106 -0
- data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb +73 -0
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb +82 -0
- data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb +59 -0
- data/lib/karafka/pro/processing/{strategies/dlq/ftr.rb → consumer_groups/virtual_partitions/distributors/consistent.rb} +18 -33
- data/lib/karafka/pro/processing/filters/base.rb +3 -61
- data/lib/karafka/pro/processing/partitioner.rb +2 -57
- data/lib/karafka/pro/processing/schedulers/base.rb +10 -6
- data/lib/karafka/pro/processing/schedulers/default.rb +6 -5
- data/lib/karafka/pro/recurring_tasks/executor.rb +1 -2
- data/lib/karafka/pro/routing/features/{active_job → consumer_groups/active_job}/builder.rb +20 -18
- data/lib/karafka/pro/routing/features/{inline_insights/config.rb → consumer_groups/active_job.rb} +5 -9
- data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/config.rb +53 -0
- data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/contracts/topic.rb +91 -0
- data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator/topic.rb +90 -0
- data/lib/karafka/pro/routing/features/consumer_groups/adaptive_iterator.rb +50 -0
- data/lib/karafka/pro/routing/features/{patterns/contracts/consumer_group.rb → consumer_groups/dead_letter_queue/contracts/topic.rb} +25 -30
- data/lib/karafka/pro/routing/features/consumer_groups/dead_letter_queue/topic.rb +70 -0
- data/lib/karafka/pro/routing/features/consumer_groups/dead_letter_queue.rb +46 -0
- data/lib/karafka/pro/routing/features/{direct_assignments → consumer_groups/delaying}/config.rb +6 -4
- data/lib/karafka/pro/routing/features/{patterns/contracts/pattern.rb → consumer_groups/delaying/contracts/topic.rb} +13 -16
- data/lib/karafka/pro/routing/features/consumer_groups/delaying/topic.rb +85 -0
- data/lib/karafka/pro/routing/features/{adaptive_iterator/config.rb → consumer_groups/delaying.rb} +8 -11
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/config.rb +46 -0
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/contracts/consumer_group.rb +68 -0
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/contracts/topic.rb +125 -0
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb +97 -0
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/topic.rb +97 -0
- data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments.rb +44 -0
- data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/config.rb +51 -0
- data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/contracts/topic.rb +58 -0
- data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/topic.rb +80 -0
- data/lib/karafka/pro/routing/features/consumer_groups/inline_insights.rb +45 -0
- data/lib/karafka/pro/routing/features/consumer_groups/long_running_job/config.rb +47 -0
- data/lib/karafka/pro/routing/features/{multiplexing/patches/contracts/consumer_group.rb → consumer_groups/long_running_job/contracts/topic.rb} +13 -15
- data/lib/karafka/pro/routing/features/consumer_groups/long_running_job/topic.rb +72 -0
- data/lib/karafka/pro/routing/features/{long_running_job.rb → consumer_groups/long_running_job.rb} +6 -4
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/config.rb +58 -0
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/contracts/routing.rb +83 -0
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/contracts/topic.rb +148 -0
- data/lib/karafka/pro/routing/features/{delaying/contracts/topic.rb → consumer_groups/multiplexing/patches/contracts/consumer_group.rb} +19 -14
- data/lib/karafka/pro/routing/features/{multiplexing/subscription_group.rb → consumer_groups/multiplexing/proxy.rb} +17 -23
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/subscription_group.rb +68 -0
- data/lib/karafka/pro/routing/features/{multiplexing → consumer_groups/multiplexing}/subscription_groups_builder.rb +16 -14
- data/lib/karafka/pro/routing/features/consumer_groups/multiplexing.rb +85 -0
- data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job/topic.rb +51 -0
- data/lib/karafka/pro/routing/features/{non_blocking_job.rb → consumer_groups/non_blocking_job.rb} +15 -13
- data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/config.rb +52 -0
- data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/contracts/topic.rb +59 -0
- data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb +93 -0
- data/lib/karafka/pro/routing/features/{offset_metadata.rb → consumer_groups/offset_metadata.rb} +16 -14
- data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/builder.rb +74 -0
- data/lib/karafka/pro/routing/features/{multiplexing → consumer_groups/parallel_segments}/config.rb +13 -15
- data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb +110 -0
- data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/contracts/consumer_group.rb +74 -0
- data/lib/karafka/pro/routing/features/{parallel_segments → consumer_groups/parallel_segments}/topic.rb +24 -22
- data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments.rb +51 -0
- data/lib/karafka/pro/routing/features/{patterns → consumer_groups/patterns}/builder.rb +15 -13
- data/lib/karafka/pro/routing/features/{offset_metadata/contracts/topic.rb → consumer_groups/patterns/config.rb} +28 -13
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb +91 -0
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/consumer_group.rb +83 -0
- data/lib/karafka/pro/routing/features/{parallel_segments.rb → consumer_groups/patterns/contracts/pattern.rb} +24 -10
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/topic.rb +58 -0
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb +98 -0
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb +114 -0
- data/lib/karafka/pro/routing/features/{recurring_tasks/contracts/topic.rb → consumer_groups/patterns/patterns.rb} +12 -13
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/topic.rb +78 -0
- data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb +72 -0
- data/lib/karafka/pro/routing/features/consumer_groups/patterns.rb +52 -0
- data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/config.rb +56 -0
- data/lib/karafka/pro/routing/features/{direct_assignments/contracts/consumer_group.rb → consumer_groups/periodic_job/contracts/topic.rb} +17 -22
- data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb +122 -0
- data/lib/karafka/pro/routing/features/consumer_groups/periodic_job.rb +46 -0
- data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/builder.rb +150 -0
- data/lib/karafka/pro/routing/features/{swarm → consumer_groups/recurring_tasks}/config.rb +7 -8
- data/lib/karafka/pro/routing/features/{inline_insights → consumer_groups/recurring_tasks}/contracts/topic.rb +14 -13
- data/lib/karafka/pro/routing/features/{scheduled_messages → consumer_groups/recurring_tasks}/proxy.rb +6 -4
- data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks/topic.rb +72 -0
- data/lib/karafka/pro/routing/features/consumer_groups/recurring_tasks.rb +44 -0
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/builder.rb +154 -0
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/config.rb +47 -0
- data/lib/karafka/pro/routing/features/{long_running_job → consumer_groups/scheduled_messages}/contracts/topic.rb +14 -12
- data/lib/karafka/pro/routing/features/{delaying/config.rb → consumer_groups/scheduled_messages/proxy.rb} +6 -4
- data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb +72 -0
- data/lib/karafka/pro/routing/features/{recurring_tasks/proxy.rb → consumer_groups/scheduled_messages.rb} +3 -4
- data/lib/karafka/pro/routing/features/{non_blocking_job/topic.rb → consumer_groups/swarm/config.rb} +9 -8
- data/lib/karafka/pro/routing/features/consumer_groups/swarm/contracts/routing.rb +94 -0
- data/lib/karafka/pro/routing/features/consumer_groups/swarm/contracts/topic.rb +95 -0
- data/lib/karafka/pro/routing/features/consumer_groups/swarm/topic.rb +105 -0
- data/lib/karafka/pro/routing/features/consumer_groups/swarm.rb +58 -0
- data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/config.rb +69 -0
- data/lib/karafka/pro/routing/features/{parallel_segments/contracts/consumer_group.rb → consumer_groups/virtual_partitions/contracts/topic.rb} +21 -18
- data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/topic.rb +101 -0
- data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions.rb +46 -0
- data/lib/karafka/pro/routing/features/{scheduled_messages.rb → consumer_groups.rb} +3 -2
- data/lib/karafka/pro/routing/features/expiring/topic.rb +4 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +3 -3
- data/lib/karafka/pro/routing/features/pausing/topic.rb +3 -3
- data/lib/karafka/pro/routing/features/throttling/topic.rb +4 -4
- data/lib/karafka/pro/setup/defaults_injector.rb +70 -0
- data/lib/karafka/pro/swarm/liveness_listener.rb +22 -10
- data/lib/karafka/processing/consumer_groups/coordinator.rb +221 -0
- data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb +69 -0
- data/lib/karafka/processing/consumer_groups/executor.rb +220 -0
- data/lib/karafka/processing/consumer_groups/executors_buffer.rb +94 -0
- data/lib/karafka/processing/consumer_groups/expansions_selector.rb +26 -0
- data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb +47 -0
- data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb +23 -0
- data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb +132 -0
- data/lib/karafka/processing/consumer_groups/jobs/consume.rb +52 -0
- data/lib/karafka/processing/consumer_groups/jobs/eofed.rb +34 -0
- data/lib/karafka/processing/consumer_groups/jobs/idle.rb +33 -0
- data/lib/karafka/processing/consumer_groups/jobs/revoked.rb +34 -0
- data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb +32 -0
- data/lib/karafka/processing/consumer_groups/jobs_builder.rb +36 -0
- data/lib/karafka/processing/consumer_groups/partitioner.rb +28 -0
- data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb +48 -0
- data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb +25 -0
- data/lib/karafka/processing/consumer_groups/strategies/base.rb +65 -0
- data/lib/karafka/processing/consumer_groups/strategies/default.rb +218 -0
- data/lib/karafka/processing/consumer_groups/strategies/dlq.rb +157 -0
- data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb +72 -0
- data/lib/karafka/processing/consumer_groups/strategies/mom.rb +33 -0
- data/lib/karafka/processing/consumer_groups/strategy_selector.rb +53 -0
- data/lib/karafka/processing/coordinator.rb +4 -211
- data/lib/karafka/processing/coordinators_buffer.rb +4 -59
- data/lib/karafka/processing/jobs_queue.rb +12 -4
- data/lib/karafka/processing/partitioner.rb +4 -18
- data/lib/karafka/processing/schedulers/default.rb +2 -1
- data/lib/karafka/processing/strategy_selector.rb +4 -42
- data/lib/karafka/processing/worker.rb +8 -4
- data/lib/karafka/processing/workers_pool.rb +158 -0
- data/lib/karafka/routing/builder.rb +12 -12
- data/lib/karafka/routing/contracts/routing.rb +3 -4
- data/lib/karafka/routing/features/base/expander.rb +5 -5
- data/lib/karafka/routing/features/consumer_groups/active_job/builder.rb +35 -0
- data/lib/karafka/routing/features/consumer_groups/active_job/config.rb +17 -0
- data/lib/karafka/routing/features/consumer_groups/active_job/contracts/topic.rb +44 -0
- data/lib/karafka/routing/features/consumer_groups/active_job/proxy.rb +16 -0
- data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb +50 -0
- data/lib/karafka/routing/features/consumer_groups/active_job.rb +15 -0
- data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/config.rb +39 -0
- data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/contracts/topic.rb +58 -0
- data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/topic.rb +76 -0
- data/lib/karafka/routing/features/consumer_groups/dead_letter_queue.rb +18 -0
- data/lib/karafka/routing/features/consumer_groups/eofed/config.rb +17 -0
- data/lib/karafka/routing/features/consumer_groups/eofed/contracts/topic.rb +39 -0
- data/lib/karafka/routing/features/consumer_groups/eofed/topic.rb +42 -0
- data/lib/karafka/routing/features/consumer_groups/eofed.rb +16 -0
- data/lib/karafka/routing/features/consumer_groups/inline_insights/config.rb +17 -0
- data/lib/karafka/routing/features/consumer_groups/inline_insights/contracts/topic.rb +27 -0
- data/lib/karafka/routing/features/consumer_groups/inline_insights/topic.rb +42 -0
- data/lib/karafka/routing/features/consumer_groups/inline_insights.rb +42 -0
- data/lib/karafka/routing/features/consumer_groups/manual_offset_management/config.rb +17 -0
- data/lib/karafka/routing/features/consumer_groups/manual_offset_management/contracts/topic.rb +27 -0
- data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb +46 -0
- data/lib/karafka/routing/features/consumer_groups/manual_offset_management.rb +20 -0
- data/lib/karafka/routing/features/consumer_groups.rb +12 -0
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +4 -19
- data/lib/karafka/routing/features/declaratives/topic.rb +30 -14
- data/lib/karafka/routing/features/deserializers/topic.rb +3 -3
- data/lib/karafka/routing/router.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +18 -9
- data/lib/karafka/routing/topic.rb +25 -11
- data/lib/karafka/runner.rb +17 -17
- data/lib/karafka/server.rb +28 -6
- data/lib/karafka/setup/attributes_map.rb +2 -0
- data/lib/karafka/setup/config.rb +64 -15
- data/lib/karafka/setup/config_proxy.rb +1 -2
- data/lib/karafka/setup/contracts/config.rb +28 -8
- data/lib/karafka/setup/defaults_injector.rb +10 -0
- data/lib/karafka/status.rb +1 -2
- data/lib/karafka/swarm/liveness_listener.rb +7 -0
- data/lib/karafka/swarm/manager.rb +7 -7
- data/lib/karafka/swarm/node.rb +8 -0
- data/lib/karafka/swarm/supervisor.rb +9 -1
- data/lib/karafka/templates/karafka.rb.erb +11 -5
- data/lib/karafka/version.rb +1 -1
- metadata +237 -224
- data/lib/karafka/connection/rebalance_manager.rb +0 -116
- data/lib/karafka/instrumentation/callbacks/error.rb +0 -52
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +0 -84
- data/lib/karafka/instrumentation/callbacks/statistics.rb +0 -55
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +0 -79
- data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +0 -92
- data/lib/karafka/pro/processing/collapser.rb +0 -79
- data/lib/karafka/pro/processing/coordinator.rb +0 -197
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +0 -119
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +0 -152
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +0 -207
- data/lib/karafka/pro/processing/expansions_selector.rb +0 -52
- data/lib/karafka/pro/processing/filters/delayer.rb +0 -87
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +0 -95
- data/lib/karafka/pro/processing/filters/throttler.rb +0 -101
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +0 -74
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +0 -51
- data/lib/karafka/pro/processing/jobs_builder.rb +0 -90
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +0 -61
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +0 -149
- data/lib/karafka/pro/processing/offset_metadata/listener.rb +0 -63
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +0 -98
- data/lib/karafka/pro/processing/parallel_segments/filters/default.rb +0 -110
- data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb +0 -91
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +0 -85
- data/lib/karafka/pro/processing/piping/consumer.rb +0 -149
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +0 -94
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +0 -87
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +0 -85
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +0 -81
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +0 -85
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +0 -79
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +0 -84
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +0 -86
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +0 -54
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +0 -101
- data/lib/karafka/pro/processing/strategies/base.rb +0 -43
- data/lib/karafka/pro/processing/strategies/default.rb +0 -416
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +0 -280
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +0 -90
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +0 -92
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +0 -60
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +0 -58
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +0 -92
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +0 -58
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +0 -57
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +0 -95
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +0 -56
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +0 -91
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +0 -54
- data/lib/karafka/pro/processing/strategies/dlq/vp.rb +0 -57
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +0 -141
- data/lib/karafka/pro/processing/strategies/ftr/vp.rb +0 -57
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +0 -114
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +0 -89
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +0 -87
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +0 -57
- data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +0 -56
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +0 -96
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +0 -54
- data/lib/karafka/pro/processing/strategies/vp/default.rb +0 -192
- data/lib/karafka/pro/processing/strategies.rb +0 -39
- data/lib/karafka/pro/processing/strategy_selector.rb +0 -102
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +0 -68
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +0 -77
- data/lib/karafka/pro/routing/features/active_job.rb +0 -43
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +0 -89
- data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +0 -88
- data/lib/karafka/pro/routing/features/adaptive_iterator.rb +0 -48
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +0 -74
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +0 -68
- data/lib/karafka/pro/routing/features/dead_letter_queue.rb +0 -44
- data/lib/karafka/pro/routing/features/delaying/topic.rb +0 -83
- data/lib/karafka/pro/routing/features/delaying.rb +0 -46
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +0 -123
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +0 -95
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +0 -95
- data/lib/karafka/pro/routing/features/direct_assignments.rb +0 -42
- data/lib/karafka/pro/routing/features/inline_insights/topic.rb +0 -78
- data/lib/karafka/pro/routing/features/inline_insights.rb +0 -43
- data/lib/karafka/pro/routing/features/long_running_job/config.rb +0 -45
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +0 -70
- data/lib/karafka/pro/routing/features/multiplexing/contracts/routing.rb +0 -81
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +0 -146
- data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +0 -58
- data/lib/karafka/pro/routing/features/multiplexing.rb +0 -83
- data/lib/karafka/pro/routing/features/offset_metadata/config.rb +0 -50
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +0 -91
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +0 -72
- data/lib/karafka/pro/routing/features/parallel_segments/config.rb +0 -52
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +0 -108
- data/lib/karafka/pro/routing/features/patterns/config.rb +0 -71
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +0 -89
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +0 -56
- data/lib/karafka/pro/routing/features/patterns/detector.rb +0 -96
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +0 -112
- data/lib/karafka/pro/routing/features/patterns/topic.rb +0 -76
- data/lib/karafka/pro/routing/features/patterns/topics.rb +0 -70
- data/lib/karafka/pro/routing/features/patterns.rb +0 -50
- data/lib/karafka/pro/routing/features/periodic_job/config.rb +0 -54
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +0 -59
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +0 -120
- data/lib/karafka/pro/routing/features/periodic_job.rb +0 -44
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +0 -148
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +0 -45
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +0 -70
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +0 -42
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +0 -152
- data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +0 -45
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +0 -55
- data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +0 -70
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +0 -92
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +0 -93
- data/lib/karafka/pro/routing/features/swarm/topic.rb +0 -103
- data/lib/karafka/pro/routing/features/swarm.rb +0 -56
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +0 -67
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +0 -73
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +0 -99
- data/lib/karafka/pro/routing/features/virtual_partitions.rb +0 -44
- data/lib/karafka/processing/executor.rb +0 -216
- data/lib/karafka/processing/executors_buffer.rb +0 -90
- data/lib/karafka/processing/expansions_selector.rb +0 -22
- data/lib/karafka/processing/inline_insights/consumer.rb +0 -43
- data/lib/karafka/processing/inline_insights/listener.rb +0 -19
- data/lib/karafka/processing/inline_insights/tracker.rb +0 -129
- data/lib/karafka/processing/jobs/consume.rb +0 -47
- data/lib/karafka/processing/jobs/eofed.rb +0 -29
- data/lib/karafka/processing/jobs/idle.rb +0 -31
- data/lib/karafka/processing/jobs/revoked.rb +0 -29
- data/lib/karafka/processing/jobs/shutdown.rb +0 -30
- data/lib/karafka/processing/jobs_builder.rb +0 -34
- data/lib/karafka/processing/strategies/aj_dlq_mom.rb +0 -44
- data/lib/karafka/processing/strategies/aj_mom.rb +0 -21
- data/lib/karafka/processing/strategies/base.rb +0 -61
- data/lib/karafka/processing/strategies/default.rb +0 -214
- data/lib/karafka/processing/strategies/dlq.rb +0 -153
- data/lib/karafka/processing/strategies/dlq_mom.rb +0 -68
- data/lib/karafka/processing/strategies/mom.rb +0 -29
- data/lib/karafka/processing/workers_batch.rb +0 -29
- data/lib/karafka/routing/features/active_job/builder.rb +0 -33
- data/lib/karafka/routing/features/active_job/config.rb +0 -15
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +0 -42
- data/lib/karafka/routing/features/active_job/proxy.rb +0 -14
- data/lib/karafka/routing/features/active_job/topic.rb +0 -48
- data/lib/karafka/routing/features/active_job.rb +0 -13
- data/lib/karafka/routing/features/dead_letter_queue/config.rb +0 -37
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +0 -56
- data/lib/karafka/routing/features/dead_letter_queue/topic.rb +0 -74
- data/lib/karafka/routing/features/dead_letter_queue.rb +0 -16
- data/lib/karafka/routing/features/declaratives/config.rb +0 -18
- data/lib/karafka/routing/features/eofed/config.rb +0 -15
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +0 -37
- data/lib/karafka/routing/features/eofed/topic.rb +0 -40
- data/lib/karafka/routing/features/eofed.rb +0 -14
- data/lib/karafka/routing/features/inline_insights/config.rb +0 -15
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +0 -25
- data/lib/karafka/routing/features/inline_insights/topic.rb +0 -40
- data/lib/karafka/routing/features/inline_insights.rb +0 -40
- data/lib/karafka/routing/features/manual_offset_management/config.rb +0 -15
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +0 -25
- data/lib/karafka/routing/features/manual_offset_management/topic.rb +0 -44
- data/lib/karafka/routing/features/manual_offset_management.rb +0 -18
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 04dfc2086393c6d3d1b8c78eca503c425842453163b014dcba68a5c4231fe090
|
|
4
|
+
data.tar.gz: 5f62160e469b6eb2383234f630654b46fe704e9dedd896e863771c945ab8e30e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3f1bb0b95294bb925b71ff4cd91d0603b538b4f7f9a222a4789aa9aeef89f7f80b8b26ebf2623f1c44585e1de1d87d4be6dbb55748f61de9b1dd90cab68059d7
|
|
7
|
+
data.tar.gz: 1c79e3b072d6b53c0062f0ade986f357f677cae6d476297389732c3c130fd89e84687526729ba2590763d15d5f0279e08e3ca9fceec386401b2d9d5ec3ef2096
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
# Karafka Framework Changelog
|
|
2
2
|
|
|
3
|
+
## 2.6.0 (Unreleased)
|
|
4
|
+
- [Maintenance] Replace manual per-version `Warning[:performance]` / `Warning[:deprecated]` flag setup in specs with a dynamic `Warning.categories`-based approach that enables all non-experimental warning categories automatically, keeping spec warning configuration in sync with karafka-rdkafka.
|
|
5
|
+
- [Enhancement] Replace per-partition `query_watermark_offsets` consumer calls in `Admin::Topics#read_watermark_offsets` with two batch `list_offsets` admin calls (`:earliest` and `:latest`), reducing N sequential consumer roundtrips to 2 admin roundtrips regardless of how many topics or partitions are queried. Replace per-partition `query_watermark_offsets` fallback in `Admin::ConsumerGroups#seek` (when a time-based offset resolves beyond the last message) with a single batch `list_offsets` call covering all affected partitions. Replace per-partition `query_watermark_offsets` loop in `Pro::Iterator::TplBuilder#resolve_partitions_with_negative_offsets` with one batched `offsets_for_times(epoch)` call for consumer metadata warm-up plus two batch admin `list_offsets` calls for LWM/HWM resolution, reducing N sequential calls to 3 total (Pro).
|
|
6
|
+
- **[Feature]** Add `Karafka::Admin.read_partition_offsets` (and `Admin::Topics#read_partition_offsets`) to expose the `rd_kafka_ListOffsets` admin API. Allows querying partition offsets by spec (`:earliest`, `:latest`, `:max_timestamp`, or a timestamp in ms) without a consumer group. Accepts an optional `isolation_level:` keyword — pass `Karafka::Admin::IsolationLevels::READ_COMMITTED` to get the Last Stable Offset (LSO) instead of the high-watermark, giving accurate lag figures on transactionally-produced topics.
|
|
7
|
+
- **[Feature]** Add `Karafka::Admin::IsolationLevels` module with `READ_UNCOMMITTED` and `READ_COMMITTED` constants. These wrap the underlying `Rdkafka::Bindings` integer constants so callers do not need to reference rdkafka internals directly.
|
|
8
|
+
- **[Feature]** Introduce standalone `Karafka::Declaratives` subsystem (`Declaratives::Topic`, `Declaratives::Repository`, `Declaratives::Builder`) decoupled from routing. Topic declarations can now be defined via `Karafka::App.declaratives.draw` independently of routing blocks. The existing `routing#config(partitions:, ...)` DSL continues to work as a backwards-compatible bridge that populates the shared declaratives repository.
|
|
9
|
+
- [Enhancement] CLI topic commands (`create`, `delete`, `reset`, `repartition`, `align`, `migrate`, `plan`) now read from `Karafka::App.declaratives.topics` instead of walking the routing tree. Topics declared via the standalone `declaratives.draw` DSL are now managed by all CLI commands without requiring a routing entry. The routing bridge sets `bootstrap_servers` on declarations for cluster filtering.
|
|
10
|
+
- [Enhancement] Move consumer-group-specific `Processing` components (`Coordinator`, `CoordinatorsBuffer`, `Executor`, `ExecutorsBuffer`, `Partitioner`, `ExpansionsSelector`, `InlineInsights`) and Pro equivalents (including `SubscriptionGroupsCoordinator`, `Collapser`, `AdaptiveIterator`, `PeriodicJob`, `ParallelSegments`, `VirtualPartitions`, `Filters`, `OffsetMetadata`, `Piping`) under the `ConsumerGroups` namespace (internal). The `config.processing.coordinator_class`, `executor_class`, `partitioner_class`, and `expansions_selector` defaults are updated accordingly; user-provided classes are unaffected. Generic processing components (`JobsQueue`, `Worker`, `WorkersPool`, `Schedulers::*`, `Jobs::Base`) stay in their original locations.
|
|
11
|
+
- [Enhancement] Move processing strategies (`Karafka::Processing::Strategies::*` and `Karafka::Pro::Processing::Strategies::*`) under the `ConsumerGroups` namespace (internal). Prepares the processing layer for parallel `ShareGroups::Strategies` once KIP-932 lands.
|
|
12
|
+
- [Enhancement] Move `Karafka::Processing::StrategySelector` and `Karafka::Pro::Processing::StrategySelector` under the `ConsumerGroups` namespace (internal). The `config.processing.strategy_selector` default is updated accordingly; user-provided selectors are unaffected.
|
|
13
|
+
- [Enhancement] Move `Karafka::Connection::RebalanceManager` to `Karafka::Connection::ConsumerGroups::RebalanceManager` (internal). Prepares the connection layer for a parallel `ShareGroups` namespace once KIP-932 lands.
|
|
14
|
+
- [Enhancement] Move the rebalance librdkafka callback to `Karafka::Instrumentation::Callbacks::ConsumerGroups::Rebalance` (internal). Published event names (`rebalance.*`) and payload shape are unchanged.
|
|
15
|
+
- [Enhancement] Move `Karafka::Instrumentation::Callbacks::Error` and `Karafka::Instrumentation::Callbacks::Statistics` to `Karafka::Instrumentation::Callbacks::ConsumerGroups::Error` and `ConsumerGroups::Statistics` (internal). Completes the callbacks layer namespacing — all consumer-group-specific librdkafka callbacks now live under `Callbacks::ConsumerGroups`. Published event names and payload shape are unchanged.
|
|
16
|
+
- [Enhancement] Namespace consumer-group-specific processing job classes under `Karafka::Processing::ConsumerGroups::Jobs` (OSS: `Consume`, `Eofed`, `Revoked`, `Shutdown`, `Idle`) and `Karafka::Pro::Processing::ConsumerGroups::Jobs` (Pro: `ConsumeNonBlocking`, `EofedNonBlocking`, `Periodic`, `PeriodicNonBlocking`, `RevokedNonBlocking`). Only `Jobs::Base` stays under `Karafka::Processing::Jobs` as the generic base class. Internal move only — no public API changes; prepares the processing layer for parallel `ShareGroups::Jobs` (KIP-932).
|
|
17
|
+
- [Enhancement] Move `Karafka::Processing::JobsBuilder` to `Karafka::Processing::ConsumerGroups::JobsBuilder` and `Karafka::Pro::Processing::JobsBuilder` to `Karafka::Pro::Processing::ConsumerGroups::JobsBuilder` (internal). The jobs builder only creates consumer-group-specific jobs; prepares the processing layer for a parallel `ShareGroups::JobsBuilder` once KIP-932 lands.
|
|
18
|
+
- [Enhancement] Nest consumer-group-specific processing config settings (`coordinator_class`, `errors_tracker_class`, `partitioner_class`, `strategy_selector`, `expansions_selector`, `executor_class`, `jobs_builder`) under `config.internal.processing.consumer_groups`. Shared settings (`jobs_queue_class`, `scheduler_class`, `worker_job_call_wrapper`) remain at the `config.internal.processing` level. Prepares the config structure for a parallel `config.internal.processing.share_groups` sub-namespace once KIP-932 lands.
|
|
19
|
+
- [Enhancement] Move OSS consumer-group-specific routing features (`ActiveJob`, `DeadLetterQueue`, `Eofed`, `InlineInsights`, `ManualOffsetManagement`) from `Routing::Features` to `Routing::Features::ConsumerGroups` (internal). Agnostic features (`Declaratives`, `Deserializers`) remain at the `Routing::Features` level. Prepares the routing layer for a parallel `Routing::Features::ShareGroups` namespace once KIP-932 lands.
|
|
20
|
+
- [Enhancement] Move Pro consumer-group-specific routing features (`ActiveJob`, `AdaptiveIterator`, `DeadLetterQueue`, `Delaying`, `DirectAssignments`, `InlineInsights`, `LongRunningJob`, `Multiplexing`, `NonBlockingJob`, `OffsetMetadata`, `ParallelSegments`, `Patterns`, `PeriodicJob`, `RecurringTasks`, `ScheduledMessages`, `Swarm`, `VirtualPartitions`) from `Pro::Routing::Features` to `Pro::Routing::Features::ConsumerGroups` (internal). Agnostic features (`Expiring`, `Filtering`, `Pausing`, `Throttling`) remain at the `Pro::Routing::Features` level.
|
|
21
|
+
- [Enhancement] Introduce polymorphic `Routing::Topic#group` and `Routing::SubscriptionGroup#group` accessors (with `#consumer_group` kept as a backwards-compatible alias), expose `group_id` alongside `consumer_group_id` in `Routing::Topic#to_h`, emit parallel `group:` / `group_id:` keys alongside `consumer_group:` / `consumer_group_id:` in `rebalance.*` payloads, and emit parallel `group_id:` alongside `consumer_group_id:` in `error.occurred` and `statistics.emitted` payloads (these callbacks only carry the group id, not the group object). Prepares routing for additional group types (e.g. KIP-932 share groups); legacy keys remain and will be retired in Karafka 3.0 once share groups land.
|
|
22
|
+
- **[Feature]** Add `Processing::WorkersPool` with dynamic thread pool scaling via `#scale`, `nil` sentinel-based worker exit for downscaling, and `worker.scaling.up`/`worker.scaling.down` instrumentation events.
|
|
23
|
+
- [Enhancement] Add per-partition generation tracking to `AssignmentsTracker` to distinguish first-time assignments from reassignments.
|
|
24
|
+
- [Enhancement] Make liveness listeners fiber-safe.
|
|
25
|
+
- [Fix] Include orphaned node detection in Pro swarm liveness listener.
|
|
26
|
+
- [Fix] Report liveness on `connection.listener.before_fetch_loop` so nodes send an initial healthy report before the first consumption.
|
|
27
|
+
- [Fix] Fire liveness events during `wait_pinging` so nodes continue reporting health status during shutdown with active LRJ jobs.
|
|
28
|
+
- [Maintenance] Use namespaced topic naming format in all integration specs for consistent traceability.
|
|
29
|
+
- [Fix] Fix `DataCollector::SPEC_HASH` producing non-deterministic hashes for pristine and poro specs by passing the original spec path via `KARAFKA_SPEC_PATH` env var.
|
|
30
|
+
- [Fix] Fix swarm supervisor leaking reader pipe file descriptors on node restarts by closing the old reader before creating a new pipe in `Node#start`.
|
|
31
|
+
- [Maintenance] Add `bin/tests_topics_hashes` script for looking up spec files by their topic name hash prefix.
|
|
32
|
+
- [Change] Require `karafka-rdkafka` `>=` `0.27.1` to pick up the fix for `poll_batch` and `poll_batch_nb` raising `RdkafkaError` with only the integer error code, which discarded topic/partition/offset context from `e.details` and caused `partition_eof` handling to call `@buffer.eof(nil, nil)`, resulting in a `TopicNotFoundError` crash.
|
|
33
|
+
- [Fix] Raise `InvalidLicenseTokenError` when a manually-defined `Karafka::License` module is missing required methods (`#token` or `#version`) instead of silently skipping pro component loading, which previously caused confusing `NameError` exceptions later in the boot process.
|
|
34
|
+
- [Enhancement] Bump swarm supervisor `SHUTDOWN_GRACE_PERIOD` from 1s to 15s to give forked nodes enough time to finish post-`shutdown_timeout` cleanup (at_exit handlers, librdkafka finalization, connection pool close) before the supervisor forcefully terminates them, especially on CI where `sleep` granularity and `waitpid` cost stretch each supervision loop iteration.
|
|
35
|
+
- [Maintenance] Fix flaky `pro/admin/recovery/coordinator_for_spec.rb` by warming up the `__consumer_offsets` internal topic with a produce + `seek_consumer_group` and retrying the first `Recovery.coordinator_for` call with exponential backoff on `MetadataError`.
|
|
36
|
+
|
|
3
37
|
## 2.5.9 (2026-03-30)
|
|
4
38
|
- [Enhancement] Validate that `statistics.interval.ms` is not zero when dynamic multiplexing is enabled (Pro).
|
|
5
39
|
- [Fix] Fix swarm liveness reporting to also use `on_connection_listener_fetch_loop` so nodes stay alive when `statistics.interval.ms` is disabled.
|
data/certs/expired.txt
CHANGED
|
@@ -1,2 +1,85 @@
|
|
|
1
1
|
# Expired/revoked license checksums (SHA256 of license token)
|
|
2
2
|
# One checksum per line, lines starting with # are comments
|
|
3
|
+
|
|
4
|
+
01603767339fd25cf874e6fe29315fb27b6f6bbdf2e769c3b88e6efcf4096d37
|
|
5
|
+
024e2c943693c4ea9eea852eb02d32acc9cb34df21aea90af5ab39dcf3b1d8cf
|
|
6
|
+
0371303ba704d8f256b40932aa7840a1adb1cdc6fd032ac902794320d0d35ff1
|
|
7
|
+
051dc2d8176df8d3bde1595e01af786a3f58ccab4cc48743d624e68125d80bd6
|
|
8
|
+
06e3d91c9881ebf9686f3492e53400adc2c6a5fd4a409f6775cd36094b24f181
|
|
9
|
+
0ae8daf74b8ccd6c4828ed707edf0d05df3e7d328e91ddc973e160ae8b120787
|
|
10
|
+
0b4420e202d9c6075bb108ac4bd5917e35e89aa7d1741afaab93ca07839d06fc
|
|
11
|
+
11936f8e0637565a309157d1fd4e84a3c44e5a3f9f226435e7d38bc43e452db6
|
|
12
|
+
122ecfd0493596239995b9e7425046737b4893e28e33f31815eb4727c3f607d9
|
|
13
|
+
17c14c38e10d0d2bd4a9c847a2c23fd0726459337af1d9815b6e34d1fd9af8be
|
|
14
|
+
227f48f0b22974c43f120de28d5ca33a77e58d33064f9a73748501ba700f1474
|
|
15
|
+
24d59b10d982e5642dfda3ec54cd56dda9022927c89584bbe12d31f3df28e17d
|
|
16
|
+
2634761cc75479548003f941346f0a51a6c535415696f386b40ae28f88f63efa
|
|
17
|
+
284967207a93e9c989d05ad6e5bb0dbe3e0a8a871c3fba48324558f127166bda
|
|
18
|
+
2e2b4ccd902e43feb418cc8b3aed5874ec194ae8495ded864aff917538d91f7a
|
|
19
|
+
39aeae710122bf14729a05f24f5bd702d2a1127e5c7fb9bd90de0e20890575dc
|
|
20
|
+
3ada575caaf0b8bdeca0686a93074de9ed830751e6f46251849475f70e84e408
|
|
21
|
+
4367e57ca36a4b447f23745d76611514df61b56611225be4c8f9a90560bbace9
|
|
22
|
+
47380207ca3041b14c7d4c81a69f820852069ecab4d80a068d39133f61a6e641
|
|
23
|
+
4d49b497ece81604358afc8e9adc7d1163995c4c08446bcd5c0bbfe3d7ebc1e7
|
|
24
|
+
4d97c95c46e51105705964aecbfa5219120295e7d0a172f5c098d12c3a5cbf7e
|
|
25
|
+
60171be4b327d48b2445b00f1fca3d2a3554d06c69f63f3e97742b29030a0a69
|
|
26
|
+
630907de3c6bf258bfc2c51d768d51ca446ebd49fb29b29a0b24d363d46d1a6e
|
|
27
|
+
68db4a9700f6cc3494f8eac1e0df2471f11ef7312e7b58dae839b193e226fae5
|
|
28
|
+
6a06fb85cee01a27a5e819e464b0b5fd84051d234540d11774bc17e90c91abd4
|
|
29
|
+
6d79bcac25c0026dad0f1871ccae79fc07d696df2e0f080ce1d1038454545a05
|
|
30
|
+
6e2ed71a0925d59815c49ea1e72dfd0ba28bdbb57f93d81fa34a31c50db53830
|
|
31
|
+
711a3cced55062707b64f6d652e107d0f1318fd9f3762c732fad23d04884cdd2
|
|
32
|
+
71c2d2ab89a11010c41c9938fbc2565297364b4e8378e78ddf58b22ea2c43ca2
|
|
33
|
+
72b19e6e274b1f36799eea65b07f4849db44c82315647266608ebdb6fde0cad6
|
|
34
|
+
731c0d75d9b47cca49d9d95fc8d291fda452383f920e70849373959892443329
|
|
35
|
+
77b6974b96701d82a0670c79830543e345bf0a00d9e5b5a1bbf8443439d75987
|
|
36
|
+
77e461a9a4534870cfb7cca350d56ed0a504643bb39eeb750b338179d05069a9
|
|
37
|
+
7c06718c4314e28561f337b3ecc66fb012c2510f1ed5fcb82dbab5ba009a8f7d
|
|
38
|
+
7ca5a6183ed9c97e4da2c7030f809bfb32da49319d106377bd0e6d7d52734141
|
|
39
|
+
7ce4cade640a4f34e2909360c4c0ceb8a647b7bfead7a57880d8ce2bf6eea26c
|
|
40
|
+
7d3a2cb0ab08f2f25324124973370da30210dab04abaeeed97192576238a03e0
|
|
41
|
+
7dc790fd8bf18e663c2e4276f4362b0c14795945b508807ef03a92ece266cff8
|
|
42
|
+
81e69732cc6e9c5266fffea08fd1c69fd12c1d841b979723a77533c5efef244e
|
|
43
|
+
839f4a449e0d68acccc0f6a2d0209d2d137375d740ae6d0244dde17e63131fbb
|
|
44
|
+
85d26e4fedf364e20ea33c1b0102d34cea19156243fe781ddf597f6677cf2de3
|
|
45
|
+
8609d26341a5c4e908af8b63bcc6895d99d8cc13f0403dc7b36a2920bb8e048a
|
|
46
|
+
8d3e09a94bd00710a49b186be832f74cc3340ec3d87e780e10159e5e8d048b81
|
|
47
|
+
8e933669352c272935c310d7ce798b3ea2687611378b978e412491e7fddfce4d
|
|
48
|
+
8f3190524285a5c0a0d1e24bdc2a5c417416c2cb92b9da26f0e492aaa4600bd7
|
|
49
|
+
9221a49b42e05d39e1939cf9353805abea2042457c2247859e3274976ac2fbc6
|
|
50
|
+
96c8dde1a7f938899f3f5bd764eff295f090e1c40635e7efd25d69ef55881db2
|
|
51
|
+
9b0b791259e86740d4a2cab736802614938b7043479ac8770c570b1b37f38fb7
|
|
52
|
+
a0db6c5e7c8714a2d33adff3b69d2d688d5a34e6d5ba871d1cbdfcba3edf1e89
|
|
53
|
+
a7ce882d45a00ad82e653b60d14f1a1ebefdc9033d6cc60ed0a2ed507949a9a2
|
|
54
|
+
ac2941c112f397543fde83cc1b0dcf70ec34d77e149d3e90ad07a0f1bc809933
|
|
55
|
+
adea773fadaaaeadf0f20643e84ae6c40bcb55fb73ecee5894a41836453e4abb
|
|
56
|
+
b1411e92ad939b59c898e78520c8ec82547950493a45004b747f12820aa7d390
|
|
57
|
+
b3984622fbc6ba91a2180283a6ada7e1bbe01fb705dbaae37c831e9c893b8e00
|
|
58
|
+
b5ed1ff168e86300735e59dc2786be6d535fc9e0f3c6c8a76bc4354cbbed4ae0
|
|
59
|
+
b9e38e9c1b34db847a4b82ae35382b06e61a5872aa8c24f988d7d5246ded4648
|
|
60
|
+
bdc4f155153a59c25c7c52932f77973487d6336e99eb0beeb90edf4a4120a94a
|
|
61
|
+
bf6c5c58b02fa5e7c7dc845213ad943c5b6613f12ac6012be00b1484bb85aa04
|
|
62
|
+
c011c3d168a159c135ef35bf55121edaf1a4cc94f663a1f461595776b0313160
|
|
63
|
+
c311cb6cfb38a8b2e83b3080c1f87444a52f3f645be8337559a123aaacf1344a
|
|
64
|
+
c87222a94f98c3a2191dda9254bd6ed031fd8ca190f75b40c67a0fcd4b56f795
|
|
65
|
+
ca53af70cd7e67abbb9dbf0a8da6788a9b2192aa3a5a1dd8720aaa68a2511722
|
|
66
|
+
d02da0cbd2c1d83432a2cee7984dd311249692373554002d1d543dd5007f6a58
|
|
67
|
+
d13295e597120b88bf4c4f36de63d0aa3e5a68e3578c09a2e026bd1f6aaaf04f
|
|
68
|
+
d862dd776d5b42a73a63574dc00187c587403101cd211ed348fc422f6460844e
|
|
69
|
+
d9f25b56eae6a62b9383c068c2a7c22ef6b051f1cf4137164ba3dd1baf0bac61
|
|
70
|
+
db1f2c50f3dc6956c498b06f67f82412a76f0b138265a55713076b2a4a35725f
|
|
71
|
+
ded7405c91e9bb1994c58c728f83b705fdc5e816ce0cb3fdf16887127fff27f0
|
|
72
|
+
dfe08da10ae5e74e9bbf648261860f3b94201a11ab4269cd4db7c73017452dde
|
|
73
|
+
dff7372b5ee3136af892561507602423bbae42d12d3a65277ae2af1add666723
|
|
74
|
+
e1a040c0c9a6c5e4603d9aebbc4201075238b47446348ab635b53704b3095a63
|
|
75
|
+
e3b94496b65bb626e723637926e729a33be673fe0355d964306bde563ff6fbf7
|
|
76
|
+
e577d74987f1d31c10fc4ec67dd74084486514f844668d7d01bd4cad2d8f09c4
|
|
77
|
+
e7a91a5ce0e68b888026bbddb57c92c7d5ad61e9416c8188721eee0a8ba85bc1
|
|
78
|
+
eda2ab2c99f769d646dd6fff564af9ec6f32e4462390e3b3246bf953cd7a9c70
|
|
79
|
+
f031aa98b2375c6573f6f775f6000dc81e07b1034694742a900a9115eb124900
|
|
80
|
+
f1ade1306fd64c2e32b15e505d87c0a9ec7a8392f5bea014fc4e70acd17f9935
|
|
81
|
+
f397ccd272b6ee7b33595aa93a6e40b725af265b681efca05bb26f9a7766d6c2
|
|
82
|
+
f54723b14f7cfc4e3c88b946cab881b907dd5ac8372b4600213241ab82778979
|
|
83
|
+
f9cc7b7d9c7d81ba477a70749d166edbc2b44f1714096fce10118bb18fd9696c
|
|
84
|
+
fb417a807d7595a4aff44cc79712e9bc887181e64eb9a98fe9f200b585856ae7
|
|
85
|
+
fd5009f463fe0c38cfe66677ff9b8e5031cbc4ce354883c99c392109ae8ae3b8
|
data/config/locales/errors.yml
CHANGED
|
@@ -25,22 +25,25 @@ en:
|
|
|
25
25
|
Decrease max_wait_time or increase node_report_timeout
|
|
26
26
|
kafka_format: needs to be a filled hash
|
|
27
27
|
key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
|
|
28
|
+
managed_key_not_supported: >
|
|
29
|
+
this kafka setting is managed internally by Karafka and should not be set directly.
|
|
30
|
+
Setting it manually may cause misbehaviours and other unexpected issues
|
|
28
31
|
pause.timeout_max_timeout_vs_pause_max_timeout: pause.timeout must be less or equal to pause.max_timeout
|
|
29
32
|
shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
|
|
30
33
|
worker_thread_priority_format: must be between -3 and 3
|
|
31
34
|
|
|
32
35
|
oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
|
|
33
36
|
|
|
34
|
-
internal.processing.jobs_builder_format: cannot be nil
|
|
35
37
|
internal.processing.jobs_queue_class_format: cannot be nil
|
|
36
38
|
internal.processing.scheduler_class_format: cannot be nil
|
|
37
|
-
internal.processing.coordinator_class_format: cannot be nil
|
|
38
|
-
internal.processing.partitioner_class_format: cannot be nil
|
|
39
|
-
internal.processing.strategy_selector_format: cannot be nil
|
|
40
|
-
internal.processing.expansions_selector_format: cannot be nil
|
|
41
|
-
internal.processing.executor_class_format: cannot be nil
|
|
42
39
|
internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
|
|
43
|
-
internal.processing.
|
|
40
|
+
internal.processing.consumer_groups.jobs_builder_format: cannot be nil
|
|
41
|
+
internal.processing.consumer_groups.coordinator_class_format: cannot be nil
|
|
42
|
+
internal.processing.consumer_groups.errors_tracker_class_format: 'needs to be nil or a class'
|
|
43
|
+
internal.processing.consumer_groups.partitioner_class_format: cannot be nil
|
|
44
|
+
internal.processing.consumer_groups.strategy_selector_format: cannot be nil
|
|
45
|
+
internal.processing.consumer_groups.expansions_selector_format: cannot be nil
|
|
46
|
+
internal.processing.consumer_groups.executor_class_format: cannot be nil
|
|
44
47
|
|
|
45
48
|
internal.active_job.dispatcher_format: cannot be nil
|
|
46
49
|
internal.active_job.job_options_contract_format: cannot be nil
|
|
@@ -173,6 +176,13 @@ en:
|
|
|
173
176
|
all topic names within a single consumer group must be unique considering namespacing styles
|
|
174
177
|
disable this validation by setting config.strict_topics_namespacing to false
|
|
175
178
|
|
|
179
|
+
declaratives:
|
|
180
|
+
topic:
|
|
181
|
+
declaratives.partitions_format: needs to be more or equal to 1
|
|
182
|
+
declaratives.active_format: needs to be either true or false
|
|
183
|
+
declaratives.replication_factor_format: needs to be more or equal to 1
|
|
184
|
+
declaratives.details_format: needs to be a hash with only symbol keys
|
|
185
|
+
|
|
176
186
|
job_options:
|
|
177
187
|
missing: needs to be present
|
|
178
188
|
dispatch_method_format: needs to be either :produce_async or :produce_sync
|
data/karafka.gemspec
CHANGED
|
@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
|
|
|
21
21
|
without having to focus on things that are not your business domain.
|
|
22
22
|
DESC
|
|
23
23
|
|
|
24
|
-
spec.add_dependency "karafka-core", ">= 2.
|
|
25
|
-
spec.add_dependency "karafka-rdkafka", ">= 0.
|
|
26
|
-
spec.add_dependency "waterdrop", ">= 2.
|
|
24
|
+
spec.add_dependency "karafka-core", ">= 2.6.0", "< 2.7.0"
|
|
25
|
+
spec.add_dependency "karafka-rdkafka", ">= 0.27.2"
|
|
26
|
+
spec.add_dependency "waterdrop", ">= 2.10.1", "< 3.0.0"
|
|
27
27
|
spec.add_dependency "zeitwerk", "~> 2.3"
|
|
28
28
|
|
|
29
29
|
spec.required_ruby_version = ">= 3.2.0"
|
|
@@ -28,8 +28,7 @@ module ActiveJob
|
|
|
28
28
|
Object
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
# Karafka adapter for enqueuing jobs
|
|
32
|
-
# This is here for ease of integration with ActiveJob.
|
|
31
|
+
# Karafka adapter for enqueuing jobs This is here for ease of integration with ActiveJob.
|
|
33
32
|
class KarafkaAdapter < base
|
|
34
33
|
include ::Karafka::Helpers::ConfigImporter.new(
|
|
35
34
|
dispatcher: %i[internal active_job dispatcher]
|
|
@@ -19,8 +19,7 @@ module Karafka
|
|
|
19
19
|
def karafka_options(new_options = {})
|
|
20
20
|
return _karafka_options if new_options.empty?
|
|
21
21
|
|
|
22
|
-
# Make sure, that karafka options that someone wants to use are valid before assigning
|
|
23
|
-
# them
|
|
22
|
+
# Make sure, that karafka options that someone wants to use are valid before assigning them
|
|
24
23
|
App.config.internal.active_job.job_options_contract.validate!(
|
|
25
24
|
new_options,
|
|
26
25
|
scope: %w[active_job]
|
|
@@ -76,8 +76,7 @@ module Karafka
|
|
|
76
76
|
|
|
77
77
|
private
|
|
78
78
|
|
|
79
|
-
# Recognizes whether the type is provided and remaps it to a symbol representation if
|
|
80
|
-
# needed
|
|
79
|
+
# Recognizes whether the type is provided and remaps it to a symbol representation if needed
|
|
81
80
|
#
|
|
82
81
|
# @param type [Symbol, Integer]
|
|
83
82
|
# @return [Symbol]
|