karafka 2.5.4 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/LICENSE-COMM +8 -2
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +128 -81
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +84 -70
- data/lib/karafka/admin/consumer_groups.rb +377 -330
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +304 -280
- data/lib/karafka/admin/topics.rb +232 -186
- data/lib/karafka/admin.rb +276 -117
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +53 -53
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +28 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/consumer.rb +19 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +19 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +21 -4
- data/lib/karafka/pro/base_consumer.rb +19 -2
- data/lib/karafka/pro/cleaner/errors.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/message.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +29 -2
- data/lib/karafka/pro/cleaner/messages/metadata.rb +19 -2
- data/lib/karafka/pro/cleaner.rb +19 -2
- data/lib/karafka/pro/cli/contracts/server.rb +31 -14
- data/lib/karafka/pro/cli/parallel_segments/base.rb +23 -6
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +24 -7
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +22 -5
- data/lib/karafka/pro/cli/parallel_segments.rb +26 -9
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +33 -16
- data/lib/karafka/pro/connection/multiplexing/listener.rb +19 -2
- data/lib/karafka/pro/contracts/base.rb +19 -2
- data/lib/karafka/pro/encryption/cipher.rb +19 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +21 -4
- data/lib/karafka/pro/encryption/errors.rb +19 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +21 -4
- data/lib/karafka/pro/encryption/messages/parser.rb +21 -4
- data/lib/karafka/pro/encryption/setup/config.rb +21 -4
- data/lib/karafka/pro/encryption.rb +19 -2
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +19 -2
- data/lib/karafka/pro/iterator/expander.rb +19 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +21 -4
- data/lib/karafka/pro/iterator.rb +20 -3
- data/lib/karafka/pro/loader.rb +21 -3
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +20 -3
- data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +19 -2
- data/lib/karafka/pro/processing/collapser.rb +19 -2
- data/lib/karafka/pro/processing/coordinator.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +26 -9
- data/lib/karafka/pro/processing/executor.rb +19 -2
- data/lib/karafka/pro/processing/expansions_selector.rb +19 -2
- data/lib/karafka/pro/processing/filters/base.rb +20 -3
- data/lib/karafka/pro/processing/filters/delayer.rb +21 -4
- data/lib/karafka/pro/processing/filters/expirer.rb +19 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +20 -3
- data/lib/karafka/pro/processing/filters/throttler.rb +19 -2
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +19 -2
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs_builder.rb +19 -2
- data/lib/karafka/pro/processing/jobs_queue.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +20 -3
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/listener.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +25 -8
- data/lib/karafka/pro/processing/parallel_segments/filters/default.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb +19 -2
- data/lib/karafka/pro/processing/partitioner.rb +22 -5
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +25 -7
- data/lib/karafka/pro/processing/piping/consumer.rb +26 -9
- data/lib/karafka/pro/processing/schedulers/base.rb +24 -7
- data/lib/karafka/pro/processing/schedulers/default.rb +24 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +21 -4
- data/lib/karafka/pro/processing/strategies/aj/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/base.rb +19 -2
- data/lib/karafka/pro/processing/strategies/default.rb +41 -24
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +26 -9
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/ftr/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/vp/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies.rb +19 -2
- data/lib/karafka/pro/processing/strategy_selector.rb +19 -2
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/base.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/errors.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/executor.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +20 -3
- data/lib/karafka/pro/recurring_tasks/matcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/schedule.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +24 -7
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +22 -5
- data/lib/karafka/pro/recurring_tasks/task.rb +23 -6
- data/lib/karafka/pro/recurring_tasks.rb +23 -6
- data/lib/karafka/pro/routing/features/active_job/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/active_job.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator.rb +19 -2
- data/lib/karafka/pro/routing/features/base.rb +19 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/dead_letter_queue.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/config.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/direct_assignments.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/config.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/config.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/config.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/inline_insights/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/long_running_job.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +24 -7
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/non_blocking_job.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/config.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/offset_metadata.rb +20 -3
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +24 -7
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/parallel_segments/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/config.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +22 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/detector.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/patterns/topics.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/pausing/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/periodic_job.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +26 -9
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +32 -15
- data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/config.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm.rb +20 -3
- data/lib/karafka/pro/routing/features/throttling/config.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/throttling/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +26 -9
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +29 -12
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/day.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +24 -7
- data/lib/karafka/pro/scheduled_messages/errors.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/proxy.rb +27 -10
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/serializer.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/state.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/tracker.rb +21 -4
- data/lib/karafka/pro/scheduled_messages.rb +21 -4
- data/lib/karafka/pro/swarm/liveness_listener.rb +21 -4
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +5 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +7 -7
- data/lib/karafka/swarm/supervisor.rb +6 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11edff86c8615652130786431e179d242dcc33130e5aabbbf5e0f5ed6d4138fe
|
|
4
|
+
data.tar.gz: 69244021709283a153da19907a424c230a3597832beec6acbb39dbc02e738256
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 843548637c77ace03cde5c1f4150226c244b1b3343c6acd17d16c19cdc798c6d770611522eed7ebef76830f3e551c75afa176f2937c1fd63045e66ccb6276701
|
|
7
|
+
data.tar.gz: a0696c28b998c8e13d4b0e0d3f4252245f27e6f93e0d75d2153a93cc1dcc53b45fbe8416ba0a9e6025753ec5d107088d578f03d0a4cada3bb5ed2eb4e5341518
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Karafka Framework Changelog
|
|
2
2
|
|
|
3
|
+
## 2.5.6 (2026-02-28)
|
|
4
|
+
- **[Feature]** Add `karafka topics health` command to check Kafka topics for replication and durability issues, detecting no redundancy (RF=1), zero fault tolerance (RF≤min.insync), and low durability (min.insync=1) configurations with color-coded severity grouping and actionable recommendations (Pro).
|
|
5
|
+
- [Enhancement] Optimize license loading process by reading license files directly from the gem directory instead of requiring the entire gem, reducing initialization overhead and adding support for user-defined License modules.
|
|
6
|
+
- [Change] Migrate Admin `AbstractHandle#wait` calls from deprecated `max_wait_timeout` (seconds) to `max_wait_timeout_ms` (milliseconds) to align with `karafka-rdkafka` `0.24.0`.
|
|
7
|
+
- [Change] Require `karafka-rdkafka` `>=` `0.24.0` to support the new `max_wait_timeout_ms` API.
|
|
8
|
+
- [Fix] Fix LRJ partition freeze when filter returns `applied? true` with `action :skip`, causing partition to remain paused for ~31,709 years instead of resuming after processing.
|
|
9
|
+
|
|
10
|
+
## 2.5.5 (2026-01-24)
|
|
11
|
+
- [Feature] Add multi-cluster Admin support via `Karafka::Admin.new(kafka: { ... })` allowing operations against different Kafka clusters while maintaining backward compatibility with existing class-method API.
|
|
12
|
+
|
|
3
13
|
## 2.5.4 (2025-12-28)
|
|
4
14
|
- [Enhancement] Align Datadog logger listener error handling with the main logger listener by adding missing error types and replacing `UnsupportedCaseError` with generic error logging to support dynamic errors reporting.
|
|
5
15
|
- [Enhancement] Align Appsignal metrics listener `USER_CONSUMER_ERROR_TYPES` with all consumer error types for complete error tracking.
|
data/LICENSE-COMM
CHANGED
|
@@ -12,6 +12,10 @@ In order to use the Software under this Agreement, you must either: (a) receive
|
|
|
12
12
|
|
|
13
13
|
1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold.
|
|
14
14
|
|
|
15
|
+
1.1.1 Entity Definition. Where you have purchased or otherwise acquired the Software as or for an entity, "you" and "your" refer exclusively to the single legal entity (such as a corporation, limited liability company, partnership, or other legally recognized organization) that is the named licensee. This license does not extend to any parent company, subsidiary, affiliate, related entity, or other member of a corporate group, regardless of ownership structure or control relationships. Each separate legal entity requires its own license.
|
|
16
|
+
|
|
17
|
+
1.1.2 Mergers, Acquisitions, and Change of Control. In the event of a merger, acquisition, consolidation, or other change of control affecting the licensee entity: (a) if the licensee entity ceases to exist as a separate legal entity, this license terminates and the surviving or acquiring entity must purchase a new license or obtain prior written consent from Maciej Mensfeld to continue use of the Software; (b) if the licensee entity continues to exist as a separate legal entity (including as a wholly owned subsidiary), this license remains valid solely for that entity and does not extend to the parent, acquirer, or any affiliated entities unless explicitly agreed in writing by Maciej Mensfeld.
|
|
18
|
+
|
|
15
19
|
1.2 Pro License. If you purchased a Pro License (included with the Karafka Pro Software), you may install the Software on an unlimited number of Hosts. "Host" means any physical or virtual machine which is controlled by you. You may also run an unlimited number of Workers. "Worker" means a thread within a Karafka server process which executes jobs. You may concurrently run the software on an unlimited number of Hosts, with each host running an unlimited number of Workers.
|
|
16
20
|
|
|
17
21
|
1.3 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.
|
|
@@ -22,10 +26,12 @@ In order to use the Software under this Agreement, you must either: (a) receive
|
|
|
22
26
|
|
|
23
27
|
3. Restricted Uses.
|
|
24
28
|
|
|
25
|
-
3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 2), or incorporate the Software, except to the extent expressly authorized in writing by Maciej Mensfeld; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Maciej Mensfeld in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Maciej Mensfeld; (j) use the Software to develop a product which is competitive with any Maciej Mensfeld product offerings;
|
|
29
|
+
3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 2), or incorporate the Software, except to the extent expressly authorized in writing by Maciej Mensfeld; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Maciej Mensfeld in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Maciej Mensfeld; (j) use the Software to develop a product which is competitive with any Maciej Mensfeld product offerings; (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Maciej Mensfeld in writing. If your unique Source URL or the offline license is ever published, Maciej Mensfeld reserves the right to terminate your access without notice; (l) use the Software, its source code, documentation, or any portion thereof as training data, input, or reference material for any machine learning system, artificial intelligence model, neural network, large language model, code generation tool, or similar automated system, whether for commercial or non-commercial purposes; (m) include the Software, its source code, or any portion thereof in any dataset, corpus, or collection intended for machine learning, artificial intelligence training, or automated code analysis, regardless of whether such dataset is publicly distributed or used internally; (n) use automated tools, scripts, crawlers, or similar technologies to systematically access, scrape, copy, or collect the Software or its source code, except for the limited purpose of cloning the repository for licensed use; or (o) create, distribute, or commercialize any product, service, or tool that was developed using knowledge, patterns, or code derived from the Software through machine learning or artificial intelligence systems.
|
|
26
30
|
|
|
27
31
|
3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.
|
|
28
32
|
|
|
33
|
+
3.3 ARTIFICIAL INTELLIGENCE AND MACHINE LEARNING PROHIBITION. WITHOUT LIMITING THE GENERALITY OF SECTION 3.1, THE SOFTWARE AND ITS SOURCE CODE MAY NOT BE USED IN ANY MANNER TO TRAIN, DEVELOP, IMPROVE, OR VALIDATE ANY ARTIFICIAL INTELLIGENCE OR MACHINE LEARNING SYSTEM. THIS PROHIBITION APPLIES REGARDLESS OF WHETHER THE SOFTWARE IS ACCESSED DIRECTLY, THROUGH AUTOMATED MEANS, OR VIA THIRD-PARTY DATASETS OR SERVICES. ANY SUCH USE CONSTITUTES A MATERIAL BREACH OF THIS AGREEMENT AND MAY RESULT IN IMMEDIATE TERMINATION OF YOUR LICENSE AND PURSUIT OF ALL AVAILABLE LEGAL REMEDIES. For clarity, this prohibition applies to the training and development of AI systems, and does not restrict licensed users from utilizing AI-powered development tools in the ordinary course of developing software that integrates with the Software.
|
|
34
|
+
|
|
29
35
|
The Open Source version of the Software ("LGPL Version") is licensed under the terms of the GNU Lesser General Public License version 3.0 ("LGPL") and not under this EULA.
|
|
30
36
|
|
|
31
37
|
4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Maciej Mensfeld and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words "purchase", "sale" or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise.
|
|
@@ -36,7 +42,7 @@ The Open Source version of the Software ("LGPL Version") is licensed under the t
|
|
|
36
42
|
|
|
37
43
|
7. Term of Agreement.
|
|
38
44
|
|
|
39
|
-
7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the "Term"). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Maciej Mensfeld may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.
|
|
45
|
+
7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the "Term"). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Maciej Mensfeld may terminate this Agreement immediately upon any breach of Section 3 (including, without limitation, any use of the Software for artificial intelligence or machine learning purposes as described in Section 3.3) or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise.
|
|
40
46
|
|
|
41
47
|
7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof.
|
|
42
48
|
|
data/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-

|
|
2
2
|
|
|
3
3
|
[](https://github.com/karafka/karafka/actions/workflows/ci_linux_ubuntu_x86_64_gnu.yml)
|
|
4
4
|
[](http://badge.fury.io/rb/karafka)
|
|
@@ -39,7 +39,7 @@ Karafka **uses** threads to handle many messages simultaneously in the same proc
|
|
|
39
39
|
|
|
40
40
|
## Getting started
|
|
41
41
|
|
|
42
|
-

|
|
43
43
|
|
|
44
44
|
If you're entirely new to the subject, you can start with our "Kafka on Rails" articles series, which will get you up and running with the terminology and basic ideas behind using Kafka:
|
|
45
45
|
|
data/certs/expired.txt
ADDED
data/karafka.gemspec
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
lib = File.expand_path(
|
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
5
5
|
|
|
6
|
-
require
|
|
6
|
+
require "karafka/version"
|
|
7
7
|
|
|
8
8
|
Gem::Specification.new do |spec|
|
|
9
|
-
spec.name
|
|
10
|
-
spec.version
|
|
11
|
-
spec.platform
|
|
12
|
-
spec.authors
|
|
13
|
-
spec.email
|
|
14
|
-
spec.homepage
|
|
15
|
-
spec.licenses
|
|
16
|
-
spec.summary
|
|
9
|
+
spec.name = "karafka"
|
|
10
|
+
spec.version = Karafka::VERSION
|
|
11
|
+
spec.platform = Gem::Platform::RUBY
|
|
12
|
+
spec.authors = ["Maciej Mensfeld"]
|
|
13
|
+
spec.email = %w[contact@karafka.io]
|
|
14
|
+
spec.homepage = "https://karafka.io"
|
|
15
|
+
spec.licenses = %w[LGPL-3.0-only Commercial]
|
|
16
|
+
spec.summary = "Karafka is Ruby and Rails efficient Kafka processing framework."
|
|
17
17
|
spec.description = <<-DESC
|
|
18
18
|
Karafka is Ruby and Rails efficient Kafka processing framework.
|
|
19
19
|
|
|
@@ -21,12 +21,12 @@ Gem::Specification.new do |spec|
|
|
|
21
21
|
without having to focus on things that are not your business domain.
|
|
22
22
|
DESC
|
|
23
23
|
|
|
24
|
-
spec.add_dependency
|
|
25
|
-
spec.add_dependency
|
|
26
|
-
spec.add_dependency
|
|
27
|
-
spec.add_dependency
|
|
24
|
+
spec.add_dependency "karafka-core", ">= 2.5.6", "< 2.6.0"
|
|
25
|
+
spec.add_dependency "karafka-rdkafka", ">= 0.24.0"
|
|
26
|
+
spec.add_dependency "waterdrop", ">= 2.8.14", "< 3.0.0"
|
|
27
|
+
spec.add_dependency "zeitwerk", "~> 2.3"
|
|
28
28
|
|
|
29
|
-
spec.required_ruby_version =
|
|
29
|
+
spec.required_ruby_version = ">= 3.2.0"
|
|
30
30
|
|
|
31
31
|
gem_files = %w[
|
|
32
32
|
lib/**/*
|
|
@@ -41,16 +41,16 @@ Gem::Specification.new do |spec|
|
|
|
41
41
|
|
|
42
42
|
spec.files = Dir.glob(gem_files) & `git ls-files -z`.split("\x0")
|
|
43
43
|
|
|
44
|
-
spec.executables
|
|
44
|
+
spec.executables = %w[karafka]
|
|
45
45
|
spec.require_paths = %w[lib]
|
|
46
46
|
|
|
47
47
|
spec.metadata = {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
48
|
+
"funding_uri" => "https://karafka.io/#become-pro",
|
|
49
|
+
"homepage_uri" => "https://karafka.io",
|
|
50
|
+
"changelog_uri" => "https://karafka.io/docs/Changelog-Karafka",
|
|
51
|
+
"bug_tracker_uri" => "https://github.com/karafka/karafka/issues",
|
|
52
|
+
"source_code_uri" => "https://github.com/karafka/karafka",
|
|
53
|
+
"documentation_uri" => "https://karafka.io/docs",
|
|
54
|
+
"rubygems_mfa_required" => "true"
|
|
55
55
|
}
|
|
56
56
|
end
|
data/lib/active_job/karafka.rb
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
begin
|
|
4
4
|
# Do not load active job if already loaded
|
|
5
|
-
require
|
|
5
|
+
require "active_job" unless Object.const_defined?("ActiveJob")
|
|
6
6
|
|
|
7
|
-
require_relative
|
|
7
|
+
require_relative "queue_adapters/karafka_adapter"
|
|
8
8
|
|
|
9
9
|
module ActiveJob
|
|
10
10
|
# Namespace for usage simplification outside of Rails where Railtie will not kick in.
|
|
@@ -22,11 +22,11 @@ module ActiveJob
|
|
|
22
22
|
#
|
|
23
23
|
# @see https://github.com/sidekiq/sidekiq/issues/6746 Similar issue in Sidekiq
|
|
24
24
|
base = if defined?(Rails::VERSION)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
((Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2) ? Object : AbstractAdapter)
|
|
26
|
+
else
|
|
27
|
+
# Fallback when Rails is not loaded
|
|
28
|
+
Object
|
|
29
|
+
end
|
|
30
30
|
|
|
31
31
|
# Karafka adapter for enqueuing jobs
|
|
32
32
|
# This is here for ease of integration with ActiveJob.
|
|
@@ -29,13 +29,13 @@ module Karafka
|
|
|
29
29
|
# @param job_message [Karafka::Messages::Message] message with active job
|
|
30
30
|
def consume_job(job_message)
|
|
31
31
|
with_deserialized_job(job_message) do |job|
|
|
32
|
-
tags.add(:job_class, job[
|
|
32
|
+
tags.add(:job_class, job["job_class"])
|
|
33
33
|
|
|
34
34
|
payload = { caller: self, job: job, message: job_message }
|
|
35
35
|
|
|
36
36
|
# We publish both to make it consistent with `consumer.x` events
|
|
37
|
-
Karafka.monitor.instrument(
|
|
38
|
-
Karafka.monitor.instrument(
|
|
37
|
+
Karafka.monitor.instrument("active_job.consume", payload)
|
|
38
|
+
Karafka.monitor.instrument("active_job.consumed", payload) do
|
|
39
39
|
::ActiveJob::Base.execute(job)
|
|
40
40
|
end
|
|
41
41
|
end
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
4
|
-
require_relative
|
|
5
|
-
require_relative
|
|
6
|
-
require_relative
|
|
3
|
+
require "active_support/current_attributes"
|
|
4
|
+
require_relative "current_attributes/job_wrapper"
|
|
5
|
+
require_relative "current_attributes/loading"
|
|
6
|
+
require_relative "current_attributes/persistence"
|
|
7
7
|
|
|
8
8
|
# This code is based on Sidekiqs approach to persisting current attributes
|
|
9
9
|
# @see https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/current_attributes.rb
|
|
@@ -9,8 +9,8 @@ module Karafka
|
|
|
9
9
|
class JobOptionsContract < Contracts::Base
|
|
10
10
|
configure do |config|
|
|
11
11
|
config.error_messages = YAML.safe_load_file(
|
|
12
|
-
File.join(Karafka.gem_root,
|
|
13
|
-
).fetch(
|
|
12
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
13
|
+
).fetch("en").fetch("validations").fetch("job_options")
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
optional(:dispatch_method) do |val|
|
data/lib/karafka/admin/acl.rb
CHANGED
|
@@ -106,89 +106,84 @@ module Karafka
|
|
|
106
106
|
# Class level APIs that operate on Acl instances and/or return Acl instances.
|
|
107
107
|
# @note For the sake of consistency all methods from this API return array of Acls
|
|
108
108
|
class << self
|
|
109
|
-
#
|
|
110
|
-
# @
|
|
111
|
-
# @return [Array<Acl>] created acls
|
|
109
|
+
# @param acl [Acl] ACL rule to create
|
|
110
|
+
# @see #create
|
|
112
111
|
def create(acl)
|
|
113
|
-
|
|
114
|
-
admin.create_acl(**acl.to_native_hash)
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
[acl]
|
|
112
|
+
new.create(acl)
|
|
118
113
|
end
|
|
119
114
|
|
|
120
|
-
#
|
|
121
|
-
# @
|
|
122
|
-
# @return [Array<Acl>] deleted acls
|
|
123
|
-
# @note More than one Acl may be removed if rules match that way
|
|
115
|
+
# @param acl [Acl] ACL pattern to match for deletion
|
|
116
|
+
# @see #delete
|
|
124
117
|
def delete(acl)
|
|
125
|
-
|
|
126
|
-
admin.delete_acl(**acl.to_native_hash)
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
result.deleted_acls.map do |result_acl|
|
|
130
|
-
from_rdkafka(result_acl)
|
|
131
|
-
end
|
|
118
|
+
new.delete(acl)
|
|
132
119
|
end
|
|
133
120
|
|
|
134
|
-
#
|
|
135
|
-
# @
|
|
136
|
-
# @return [Array<Acl>] described acls
|
|
121
|
+
# @param acl [Acl] ACL pattern to describe
|
|
122
|
+
# @see #describe
|
|
137
123
|
def describe(acl)
|
|
138
|
-
|
|
139
|
-
admin.describe_acl(**acl.to_native_hash)
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
result.acls.map do |result_acl|
|
|
143
|
-
from_rdkafka(result_acl)
|
|
144
|
-
end
|
|
124
|
+
new.describe(acl)
|
|
145
125
|
end
|
|
146
126
|
|
|
147
|
-
#
|
|
148
|
-
# @return [Array<Acl>] all acls
|
|
127
|
+
# @see #all
|
|
149
128
|
def all
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
)
|
|
129
|
+
new.all
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Creates (unless already present) a given ACL rule in Kafka
|
|
134
|
+
# @param acl [Acl]
|
|
135
|
+
# @return [Array<Acl>] created acls
|
|
136
|
+
def create(acl)
|
|
137
|
+
with_admin_wait do |admin|
|
|
138
|
+
admin.create_acl(**acl.to_native_hash)
|
|
161
139
|
end
|
|
162
140
|
|
|
163
|
-
|
|
141
|
+
[acl]
|
|
142
|
+
end
|
|
164
143
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
144
|
+
# Removes acls matching provide acl pattern.
|
|
145
|
+
# @param acl [Acl]
|
|
146
|
+
# @return [Array<Acl>] deleted acls
|
|
147
|
+
# @note More than one Acl may be removed if rules match that way
|
|
148
|
+
def delete(acl)
|
|
149
|
+
result = with_admin_wait do |admin|
|
|
150
|
+
admin.delete_acl(**acl.to_native_hash)
|
|
171
151
|
end
|
|
172
152
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
)
|
|
153
|
+
result.deleted_acls.map do |result_acl|
|
|
154
|
+
from_rdkafka(result_acl)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Takes an Acl definition and describes all existing Acls matching its criteria
|
|
159
|
+
# @param acl [Acl]
|
|
160
|
+
# @return [Array<Acl>] described acls
|
|
161
|
+
def describe(acl)
|
|
162
|
+
result = with_admin_wait do |admin|
|
|
163
|
+
admin.describe_acl(**acl.to_native_hash)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
result.acls.map do |result_acl|
|
|
167
|
+
from_rdkafka(result_acl)
|
|
189
168
|
end
|
|
190
169
|
end
|
|
191
170
|
|
|
171
|
+
# Returns all acls on a cluster level
|
|
172
|
+
# @return [Array<Acl>] all acls
|
|
173
|
+
def all
|
|
174
|
+
describe(
|
|
175
|
+
self.class.new(
|
|
176
|
+
resource_type: :any,
|
|
177
|
+
resource_name: nil,
|
|
178
|
+
resource_pattern_type: :any,
|
|
179
|
+
principal: nil,
|
|
180
|
+
operation: :any,
|
|
181
|
+
permission_type: :any,
|
|
182
|
+
host: "*"
|
|
183
|
+
)
|
|
184
|
+
)
|
|
185
|
+
end
|
|
186
|
+
|
|
192
187
|
attr_reader(
|
|
193
188
|
:resource_type, :resource_name, :resource_pattern_type, :principal, :host, :operation,
|
|
194
189
|
:permission_type
|
|
@@ -196,6 +191,11 @@ module Karafka
|
|
|
196
191
|
|
|
197
192
|
# Initializes a new Acl instance with specified attributes.
|
|
198
193
|
#
|
|
194
|
+
# This class serves dual purposes:
|
|
195
|
+
# 1. As an ACL rule definition when called with resource_type and other ACL parameters
|
|
196
|
+
# 2. As an admin operations instance when called with only kafka: parameter
|
|
197
|
+
#
|
|
198
|
+
# @param kafka [Hash] custom kafka configuration for admin operations (optional)
|
|
199
199
|
# @param resource_type [Symbol, Integer] Specifies the type of Kafka resource
|
|
200
200
|
# (like :topic, :consumer_group).
|
|
201
201
|
# Accepts either a symbol from RESOURCE_TYPES_MAP or a direct rdkafka numerical type.
|
|
@@ -215,24 +215,45 @@ module Karafka
|
|
|
215
215
|
#
|
|
216
216
|
# Each parameter is mapped to its corresponding value in the respective *_MAP constant,
|
|
217
217
|
# allowing usage of more descriptive Ruby symbols instead of numerical types.
|
|
218
|
+
#
|
|
219
|
+
# @example Create an ACL rule
|
|
220
|
+
# acl = Karafka::Admin::Acl.new(
|
|
221
|
+
# resource_type: :topic,
|
|
222
|
+
# resource_name: 'my-topic',
|
|
223
|
+
# resource_pattern_type: :literal,
|
|
224
|
+
# principal: 'User:my-user',
|
|
225
|
+
# operation: :read,
|
|
226
|
+
# permission_type: :allow
|
|
227
|
+
# )
|
|
228
|
+
#
|
|
229
|
+
# @example Create an admin instance for a different cluster
|
|
230
|
+
# admin = Karafka::Admin::Acl.new(kafka: { 'bootstrap.servers': 'other:9092' })
|
|
231
|
+
# admin.do_create(acl)
|
|
218
232
|
def initialize(
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
233
|
+
kafka: nil,
|
|
234
|
+
resource_type: nil,
|
|
235
|
+
resource_name: nil,
|
|
236
|
+
resource_pattern_type: nil,
|
|
237
|
+
principal: nil,
|
|
238
|
+
host: "*",
|
|
239
|
+
operation: nil,
|
|
240
|
+
permission_type: nil
|
|
226
241
|
)
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
242
|
+
# If resource_type is provided, this is an ACL rule definition
|
|
243
|
+
if resource_type
|
|
244
|
+
@resource_type = map(resource_type, RESOURCE_TYPES_MAP)
|
|
245
|
+
@resource_name = resource_name
|
|
246
|
+
@resource_pattern_type = map(resource_pattern_type, RESOURCE_PATTERNS_TYPE_MAP)
|
|
247
|
+
@principal = principal
|
|
248
|
+
@host = host
|
|
249
|
+
@operation = map(operation, OPERATIONS_MAP)
|
|
250
|
+
@permission_type = map(permission_type, PERMISSION_TYPES_MAP)
|
|
251
|
+
super(kafka: kafka || {})
|
|
252
|
+
freeze
|
|
253
|
+
else
|
|
254
|
+
# This is an admin operations instance
|
|
255
|
+
super(kafka: kafka || {})
|
|
256
|
+
end
|
|
236
257
|
end
|
|
237
258
|
|
|
238
259
|
# Converts the Acl into a hash with native rdkafka types
|
|
@@ -251,6 +272,32 @@ module Karafka
|
|
|
251
272
|
|
|
252
273
|
private
|
|
253
274
|
|
|
275
|
+
# Yields admin instance, allows to run Acl operations and awaits on the final result
|
|
276
|
+
# Makes sure that admin is closed afterwards.
|
|
277
|
+
def with_admin_wait
|
|
278
|
+
with_admin do |admin|
|
|
279
|
+
yield(admin).wait(max_wait_timeout_ms: self.class.max_wait_time)
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Takes a rdkafka Acl result and converts it into our local Acl representation. Since the
|
|
284
|
+
# rdkafka Acl object is an integer based on on types, etc we remap it into our "more" Ruby
|
|
285
|
+
# form.
|
|
286
|
+
#
|
|
287
|
+
# @param rdkafka_acl [Rdkafka::Admin::AclBindingResult]
|
|
288
|
+
# return [Acl] mapped acl
|
|
289
|
+
def from_rdkafka(rdkafka_acl)
|
|
290
|
+
self.class.new(
|
|
291
|
+
resource_type: rdkafka_acl.matching_acl_resource_type,
|
|
292
|
+
resource_name: rdkafka_acl.matching_acl_resource_name,
|
|
293
|
+
resource_pattern_type: rdkafka_acl.matching_acl_pattern_type,
|
|
294
|
+
principal: rdkafka_acl.matching_acl_principal,
|
|
295
|
+
host: rdkafka_acl.matching_acl_host,
|
|
296
|
+
operation: rdkafka_acl.matching_acl_operation,
|
|
297
|
+
permission_type: rdkafka_acl.matching_acl_permission_type
|
|
298
|
+
)
|
|
299
|
+
end
|
|
300
|
+
|
|
254
301
|
# Maps the provided attribute based on the mapping hash and if not found returns the
|
|
255
302
|
# attribute itself. Useful when converting from Acl symbol based representation to the
|
|
256
303
|
# rdkafka one.
|
|
@@ -45,7 +45,7 @@ module Karafka
|
|
|
45
45
|
# def set(name, value)
|
|
46
46
|
# @operations[0] << Config.new(name: name, value: value.to_s)
|
|
47
47
|
# end
|
|
48
|
-
default_value = op_name == :delete ?
|
|
48
|
+
default_value = (op_name == :delete) ? " = nil" : ""
|
|
49
49
|
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
50
50
|
# @param name [String] name of the config to alter
|
|
51
51
|
# @param value [String] value of the config
|