karafka 2.5.4 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/LICENSE-COMM +8 -2
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +128 -81
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +84 -70
- data/lib/karafka/admin/consumer_groups.rb +377 -330
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +304 -280
- data/lib/karafka/admin/topics.rb +232 -186
- data/lib/karafka/admin.rb +276 -117
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +53 -53
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +28 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/consumer.rb +19 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +19 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +21 -4
- data/lib/karafka/pro/base_consumer.rb +19 -2
- data/lib/karafka/pro/cleaner/errors.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/message.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +29 -2
- data/lib/karafka/pro/cleaner/messages/metadata.rb +19 -2
- data/lib/karafka/pro/cleaner.rb +19 -2
- data/lib/karafka/pro/cli/contracts/server.rb +31 -14
- data/lib/karafka/pro/cli/parallel_segments/base.rb +23 -6
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +24 -7
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +22 -5
- data/lib/karafka/pro/cli/parallel_segments.rb +26 -9
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +33 -16
- data/lib/karafka/pro/connection/multiplexing/listener.rb +19 -2
- data/lib/karafka/pro/contracts/base.rb +19 -2
- data/lib/karafka/pro/encryption/cipher.rb +19 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +21 -4
- data/lib/karafka/pro/encryption/errors.rb +19 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +21 -4
- data/lib/karafka/pro/encryption/messages/parser.rb +21 -4
- data/lib/karafka/pro/encryption/setup/config.rb +21 -4
- data/lib/karafka/pro/encryption.rb +19 -2
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +19 -2
- data/lib/karafka/pro/iterator/expander.rb +19 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +21 -4
- data/lib/karafka/pro/iterator.rb +20 -3
- data/lib/karafka/pro/loader.rb +21 -3
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +20 -3
- data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +19 -2
- data/lib/karafka/pro/processing/collapser.rb +19 -2
- data/lib/karafka/pro/processing/coordinator.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +26 -9
- data/lib/karafka/pro/processing/executor.rb +19 -2
- data/lib/karafka/pro/processing/expansions_selector.rb +19 -2
- data/lib/karafka/pro/processing/filters/base.rb +20 -3
- data/lib/karafka/pro/processing/filters/delayer.rb +21 -4
- data/lib/karafka/pro/processing/filters/expirer.rb +19 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +20 -3
- data/lib/karafka/pro/processing/filters/throttler.rb +19 -2
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +19 -2
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs_builder.rb +19 -2
- data/lib/karafka/pro/processing/jobs_queue.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +20 -3
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/listener.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +25 -8
- data/lib/karafka/pro/processing/parallel_segments/filters/default.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb +19 -2
- data/lib/karafka/pro/processing/partitioner.rb +22 -5
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +25 -7
- data/lib/karafka/pro/processing/piping/consumer.rb +26 -9
- data/lib/karafka/pro/processing/schedulers/base.rb +24 -7
- data/lib/karafka/pro/processing/schedulers/default.rb +24 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +21 -4
- data/lib/karafka/pro/processing/strategies/aj/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/base.rb +19 -2
- data/lib/karafka/pro/processing/strategies/default.rb +41 -24
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +26 -9
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/ftr/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/vp/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies.rb +19 -2
- data/lib/karafka/pro/processing/strategy_selector.rb +19 -2
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/base.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/errors.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/executor.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +20 -3
- data/lib/karafka/pro/recurring_tasks/matcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/schedule.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +24 -7
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +22 -5
- data/lib/karafka/pro/recurring_tasks/task.rb +23 -6
- data/lib/karafka/pro/recurring_tasks.rb +23 -6
- data/lib/karafka/pro/routing/features/active_job/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/active_job.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator.rb +19 -2
- data/lib/karafka/pro/routing/features/base.rb +19 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/dead_letter_queue.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/config.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/direct_assignments.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/config.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/config.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/config.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/inline_insights/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/long_running_job.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +24 -7
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/non_blocking_job.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/config.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/offset_metadata.rb +20 -3
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +24 -7
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/parallel_segments/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/config.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +22 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/detector.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/patterns/topics.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/pausing/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/periodic_job.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +26 -9
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +32 -15
- data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/config.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm.rb +20 -3
- data/lib/karafka/pro/routing/features/throttling/config.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/throttling/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +26 -9
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +29 -12
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/day.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +24 -7
- data/lib/karafka/pro/scheduled_messages/errors.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/proxy.rb +27 -10
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/serializer.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/state.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/tracker.rb +21 -4
- data/lib/karafka/pro/scheduled_messages.rb +21 -4
- data/lib/karafka/pro/swarm/liveness_listener.rb +21 -4
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +5 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +7 -7
- data/lib/karafka/swarm/supervisor.rb +6 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -4
|
@@ -35,8 +35,8 @@ module Karafka
|
|
|
35
35
|
# @note It runs in the listener loop. Should **not** be used for anything heavy or
|
|
36
36
|
# with any potential errors. Mostly for initialization of states, etc.
|
|
37
37
|
def handle_initialized
|
|
38
|
-
monitor.instrument(
|
|
39
|
-
monitor.instrument(
|
|
38
|
+
monitor.instrument("consumer.initialize", caller: self)
|
|
39
|
+
monitor.instrument("consumer.initialized", caller: self) do
|
|
40
40
|
initialized
|
|
41
41
|
end
|
|
42
42
|
end
|
|
@@ -120,22 +120,22 @@ module Karafka
|
|
|
120
120
|
# @param action [Symbol]
|
|
121
121
|
# @param block [Proc]
|
|
122
122
|
def handle_wrap(action, &block)
|
|
123
|
-
monitor.instrument(
|
|
124
|
-
monitor.instrument(
|
|
123
|
+
monitor.instrument("consumer.wrap", caller: self)
|
|
124
|
+
monitor.instrument("consumer.wrapped", caller: self) do
|
|
125
125
|
wrap(action, &block)
|
|
126
126
|
end
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
# Run the user consumption code
|
|
130
130
|
def handle_consume
|
|
131
|
-
monitor.instrument(
|
|
132
|
-
monitor.instrument(
|
|
131
|
+
monitor.instrument("consumer.consume", caller: self)
|
|
132
|
+
monitor.instrument("consumer.consumed", caller: self) do
|
|
133
133
|
consume
|
|
134
134
|
end
|
|
135
135
|
|
|
136
136
|
# Mark job as successful
|
|
137
137
|
coordinator.success!(self)
|
|
138
|
-
rescue
|
|
138
|
+
rescue => e
|
|
139
139
|
coordinator.failure!(self, e)
|
|
140
140
|
|
|
141
141
|
# Re-raise so reported in the consumer
|
|
@@ -175,8 +175,8 @@ module Karafka
|
|
|
175
175
|
|
|
176
176
|
# Runs the consumer `#eofed` method with reporting
|
|
177
177
|
def handle_eofed
|
|
178
|
-
monitor.instrument(
|
|
179
|
-
monitor.instrument(
|
|
178
|
+
monitor.instrument("consumer.eof", caller: self)
|
|
179
|
+
monitor.instrument("consumer.eofed", caller: self) do
|
|
180
180
|
eofed
|
|
181
181
|
end
|
|
182
182
|
ensure
|
|
@@ -191,8 +191,8 @@ module Karafka
|
|
|
191
191
|
|
|
192
192
|
coordinator.revoke
|
|
193
193
|
|
|
194
|
-
monitor.instrument(
|
|
195
|
-
monitor.instrument(
|
|
194
|
+
monitor.instrument("consumer.revoke", caller: self)
|
|
195
|
+
monitor.instrument("consumer.revoked", caller: self) do
|
|
196
196
|
revoked
|
|
197
197
|
end
|
|
198
198
|
ensure
|
|
@@ -201,8 +201,8 @@ module Karafka
|
|
|
201
201
|
|
|
202
202
|
# Runs the shutdown code
|
|
203
203
|
def handle_shutdown
|
|
204
|
-
monitor.instrument(
|
|
205
|
-
monitor.instrument(
|
|
204
|
+
monitor.instrument("consumer.shutting_down", caller: self)
|
|
205
|
+
monitor.instrument("consumer.shutdown", caller: self) do
|
|
206
206
|
shutdown
|
|
207
207
|
end
|
|
208
208
|
ensure
|
|
@@ -55,9 +55,9 @@ module Karafka
|
|
|
55
55
|
|
|
56
56
|
if job
|
|
57
57
|
job.wrap do
|
|
58
|
-
monitor.instrument(
|
|
58
|
+
monitor.instrument("worker.process", instrument_details)
|
|
59
59
|
|
|
60
|
-
monitor.instrument(
|
|
60
|
+
monitor.instrument("worker.processed", instrument_details) do
|
|
61
61
|
job.before_call
|
|
62
62
|
|
|
63
63
|
# If a job is marked as non blocking, we can run a tick in the job queue and if there
|
|
@@ -87,12 +87,12 @@ module Karafka
|
|
|
87
87
|
rescue Exception => e
|
|
88
88
|
# rubocop:enable Lint/RescueException
|
|
89
89
|
monitor.instrument(
|
|
90
|
-
|
|
90
|
+
"error.occurred",
|
|
91
91
|
caller: self,
|
|
92
92
|
job: job,
|
|
93
93
|
jobs_queue: @jobs_queue,
|
|
94
94
|
error: e,
|
|
95
|
-
type:
|
|
95
|
+
type: "worker.process.error"
|
|
96
96
|
)
|
|
97
97
|
ensure
|
|
98
98
|
# job can be nil when the queue is being closed
|
|
@@ -102,7 +102,7 @@ module Karafka
|
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
# Always publish info, that we completed all the work despite its result
|
|
105
|
-
monitor.instrument(
|
|
105
|
+
monitor.instrument("worker.completed", instrument_details)
|
|
106
106
|
end
|
|
107
107
|
end
|
|
108
108
|
end
|
data/lib/karafka/railtie.rb
CHANGED
|
@@ -12,17 +12,17 @@ end
|
|
|
12
12
|
|
|
13
13
|
if Karafka.rails?
|
|
14
14
|
# Load ActiveJob adapter
|
|
15
|
-
require
|
|
15
|
+
require "active_job/karafka"
|
|
16
16
|
|
|
17
17
|
# Setup env if configured (may be configured later by .net, etc)
|
|
18
|
-
ENV[
|
|
18
|
+
ENV["KARAFKA_ENV"] ||= ENV["RAILS_ENV"] if ENV.key?("RAILS_ENV")
|
|
19
19
|
|
|
20
20
|
module Karafka
|
|
21
21
|
# Railtie for setting up Rails integration
|
|
22
22
|
class Railtie < Rails::Railtie
|
|
23
23
|
railtie_name :karafka
|
|
24
24
|
|
|
25
|
-
initializer
|
|
25
|
+
initializer "karafka.active_job_integration" do
|
|
26
26
|
ActiveSupport.on_load(:active_job) do
|
|
27
27
|
# Extend ActiveJob with some Karafka specific ActiveJob magic
|
|
28
28
|
extend ::Karafka::ActiveJob::JobExtensions
|
|
@@ -32,12 +32,12 @@ if Karafka.rails?
|
|
|
32
32
|
# This lines will make Karafka print to stdout like puma or unicorn when we run karafka
|
|
33
33
|
# server + will support code reloading with each fetched loop. We do it only for karafka
|
|
34
34
|
# based commands as Rails processes and console will have it enabled already
|
|
35
|
-
initializer
|
|
35
|
+
initializer "karafka.configure_rails_logger" do
|
|
36
36
|
# Make Karafka uses Rails logger
|
|
37
37
|
::Karafka::App.config.logger = Rails.logger
|
|
38
38
|
|
|
39
39
|
next unless Rails.env.development?
|
|
40
|
-
next unless ENV.key?(
|
|
40
|
+
next unless ENV.key?("KARAFKA_CLI")
|
|
41
41
|
# If we are already publishing to STDOUT, no need to add it again.
|
|
42
42
|
# If added again, would print stuff twice
|
|
43
43
|
next if ActiveSupport::Logger.logger_outputs_to?(Rails.logger, $stdout)
|
|
@@ -46,7 +46,7 @@ if Karafka.rails?
|
|
|
46
46
|
# Inherit the logger level from Rails, otherwise would always run with the debug level
|
|
47
47
|
stdout_logger.level = Rails.logger.level
|
|
48
48
|
|
|
49
|
-
rails71plus = Rails.gem_version >= Gem::Version.new(
|
|
49
|
+
rails71plus = Rails.gem_version >= Gem::Version.new("7.1.0")
|
|
50
50
|
|
|
51
51
|
# Rails 7.1 replaced the broadcast module with a broadcast logger
|
|
52
52
|
# While 7.1 is EOL, we keep this for users who may still use it without official support
|
|
@@ -61,20 +61,20 @@ if Karafka.rails?
|
|
|
61
61
|
end
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
-
initializer
|
|
64
|
+
initializer "karafka.configure_rails_auto_load_paths" do |app|
|
|
65
65
|
# Consumers should autoload by default in the Rails app so they are visible
|
|
66
66
|
app.config.autoload_paths += %w[app/consumers]
|
|
67
67
|
end
|
|
68
68
|
|
|
69
|
-
initializer
|
|
70
|
-
rails6plus = Rails.gem_version >= Gem::Version.new(
|
|
69
|
+
initializer "karafka.require_karafka_boot_file" do |app|
|
|
70
|
+
rails6plus = Rails.gem_version >= Gem::Version.new("6.0.0")
|
|
71
71
|
|
|
72
72
|
# If the boot file location is set to "false", we should not raise an exception and we
|
|
73
73
|
# should just not load karafka stuff. Setting this explicitly to false indicates, that
|
|
74
74
|
# karafka is part of the supply chain but it is not a first class citizen of a given
|
|
75
75
|
# system (may be just a dependency of a dependency), thus railtie should not kick in to
|
|
76
76
|
# load the non-existing boot file
|
|
77
|
-
next if Karafka.boot_file.to_s ==
|
|
77
|
+
next if Karafka.boot_file.to_s == "false"
|
|
78
78
|
|
|
79
79
|
karafka_boot_file = Rails.root.join(Karafka.boot_file.to_s).to_s
|
|
80
80
|
|
|
@@ -96,7 +96,7 @@ if Karafka.rails?
|
|
|
96
96
|
end
|
|
97
97
|
end
|
|
98
98
|
|
|
99
|
-
initializer
|
|
99
|
+
initializer "karafka.configure_worker_external_executor" do |app|
|
|
100
100
|
app.config.after_initialize do
|
|
101
101
|
app_config = Karafka::App.config
|
|
102
102
|
|
|
@@ -60,14 +60,14 @@ module Karafka
|
|
|
60
60
|
# Validate consumer group settings
|
|
61
61
|
Contracts::ConsumerGroup.new.validate!(
|
|
62
62
|
consumer_group.to_h,
|
|
63
|
-
scope: [
|
|
63
|
+
scope: ["routes", consumer_group.name]
|
|
64
64
|
)
|
|
65
65
|
|
|
66
66
|
# and then its topics settings
|
|
67
67
|
consumer_group.topics.each do |topic|
|
|
68
68
|
Contracts::Topic.new.validate!(
|
|
69
69
|
topic.to_h,
|
|
70
|
-
scope: [
|
|
70
|
+
scope: ["routes", consumer_group.name, topic.name]
|
|
71
71
|
)
|
|
72
72
|
end
|
|
73
73
|
|
|
@@ -78,7 +78,7 @@ module Karafka
|
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
# Clear out the drawn routes.
|
|
81
|
-
|
|
81
|
+
alias_method :array_clear, :clear
|
|
82
82
|
private :array_clear
|
|
83
83
|
|
|
84
84
|
# Clear routes and draw them again with the given block. Helpful for testing purposes.
|
|
@@ -8,8 +8,8 @@ module Karafka
|
|
|
8
8
|
class ConsumerGroup < Karafka::Contracts::Base
|
|
9
9
|
configure do |config|
|
|
10
10
|
config.error_messages = YAML.safe_load_file(
|
|
11
|
-
File.join(Karafka.gem_root,
|
|
12
|
-
).fetch(
|
|
11
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
12
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
required(:id) { |val| val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val) }
|
|
@@ -49,18 +49,18 @@ module Karafka
|
|
|
49
49
|
names = data.fetch(:topics).map { |topic| topic[:name] }
|
|
50
50
|
names_hash = names.each_with_object({}) { |n, h| h[n] = true }
|
|
51
51
|
error_occured = false
|
|
52
|
-
namespace_chars = [
|
|
52
|
+
namespace_chars = [".", "_"].freeze
|
|
53
53
|
names.each do |n|
|
|
54
54
|
# Skip topic names that are not namespaced
|
|
55
55
|
next unless n.chars.find { |c| namespace_chars.include?(c) }
|
|
56
56
|
|
|
57
|
-
if n.chars.include?(
|
|
57
|
+
if n.chars.include?(".")
|
|
58
58
|
# Check underscore styled topic
|
|
59
|
-
underscored_topic = n.tr(
|
|
59
|
+
underscored_topic = n.tr(".", "_")
|
|
60
60
|
error_occured = names_hash[underscored_topic] ? true : false
|
|
61
61
|
else
|
|
62
62
|
# Check dot styled topic
|
|
63
|
-
dot_topic = n.tr(
|
|
63
|
+
dot_topic = n.tr("_", ".")
|
|
64
64
|
error_occured = names_hash[dot_topic] ? true : false
|
|
65
65
|
end
|
|
66
66
|
end
|
|
@@ -7,8 +7,8 @@ module Karafka
|
|
|
7
7
|
class Routing < Karafka::Contracts::Base
|
|
8
8
|
configure do |config|
|
|
9
9
|
config.error_messages = YAML.safe_load_file(
|
|
10
|
-
File.join(Karafka.gem_root,
|
|
11
|
-
).fetch(
|
|
10
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
11
|
+
).fetch("en").fetch("validations").fetch("routing")
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
# Ensures, that when declarative topics strict requirement is on, all topics have
|
|
@@ -7,8 +7,8 @@ module Karafka
|
|
|
7
7
|
class Topic < Karafka::Contracts::Base
|
|
8
8
|
configure do |config|
|
|
9
9
|
config.error_messages = YAML.safe_load_file(
|
|
10
|
-
File.join(Karafka.gem_root,
|
|
11
|
-
).fetch(
|
|
10
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
11
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
required(:deserializers) { |val| !val.nil? }
|
|
@@ -59,7 +59,7 @@ module Karafka
|
|
|
59
59
|
|
|
60
60
|
kafka = data.fetch(:kafka)
|
|
61
61
|
|
|
62
|
-
next if kafka.key?(:
|
|
62
|
+
next if kafka.key?(:"bootstrap.servers")
|
|
63
63
|
|
|
64
64
|
[[%w[kafka bootstrap.servers], :missing]]
|
|
65
65
|
end
|
|
@@ -69,7 +69,7 @@ module Karafka
|
|
|
69
69
|
next unless Karafka::App.config.strict_topics_namespacing
|
|
70
70
|
|
|
71
71
|
value = data.fetch(:name)
|
|
72
|
-
namespace_chars = [
|
|
72
|
+
namespace_chars = [".", "_"].freeze
|
|
73
73
|
namespacing_chars_count = value.chars.find_all do |c|
|
|
74
74
|
namespace_chars.include?(c)
|
|
75
75
|
end.uniq.size
|
|
@@ -11,15 +11,15 @@ module Karafka
|
|
|
11
11
|
class Topic < Karafka::Contracts::Base
|
|
12
12
|
configure do |config|
|
|
13
13
|
config.error_messages = YAML.safe_load_file(
|
|
14
|
-
File.join(Karafka.gem_root,
|
|
15
|
-
).fetch(
|
|
14
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
15
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
virtual do |data, errors|
|
|
19
19
|
next unless errors.empty?
|
|
20
20
|
next unless data[:active_job][:active]
|
|
21
21
|
# One should not define active job jobs without ActiveJob being available for usage
|
|
22
|
-
next if Object.const_defined?(
|
|
22
|
+
next if Object.const_defined?("ActiveJob::Base")
|
|
23
23
|
|
|
24
24
|
[[%i[consumer], :active_job_missing]]
|
|
25
25
|
end
|
|
@@ -37,19 +37,19 @@ module Karafka
|
|
|
37
37
|
result = super(&block)
|
|
38
38
|
|
|
39
39
|
each do |consumer_group|
|
|
40
|
-
if scope::Contracts.const_defined?(
|
|
40
|
+
if scope::Contracts.const_defined?("ConsumerGroup", false)
|
|
41
41
|
scope::Contracts::ConsumerGroup.new.validate!(
|
|
42
42
|
consumer_group.to_h,
|
|
43
|
-
scope: [
|
|
43
|
+
scope: ["routes", consumer_group.name]
|
|
44
44
|
)
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
-
next unless scope::Contracts.const_defined?(
|
|
47
|
+
next unless scope::Contracts.const_defined?("Topic", false)
|
|
48
48
|
|
|
49
49
|
consumer_group.topics.each do |topic|
|
|
50
50
|
scope::Contracts::Topic.new.validate!(
|
|
51
51
|
topic.to_h,
|
|
52
|
-
scope: [
|
|
52
|
+
scope: ["routes", consumer_group.name, topic.name]
|
|
53
53
|
)
|
|
54
54
|
end
|
|
55
55
|
end
|
|
@@ -13,35 +13,35 @@ module Karafka
|
|
|
13
13
|
class << self
|
|
14
14
|
# Extends topic and builder with given feature API
|
|
15
15
|
def activate
|
|
16
|
-
if const_defined?(
|
|
16
|
+
if const_defined?("Topic", false)
|
|
17
17
|
Topic.prepend(self::Topic)
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
if const_defined?(
|
|
20
|
+
if const_defined?("Topics", false)
|
|
21
21
|
Topics.prepend(self::Topics)
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
if const_defined?(
|
|
24
|
+
if const_defined?("ConsumerGroup", false)
|
|
25
25
|
ConsumerGroup.prepend(self::ConsumerGroup)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
if const_defined?(
|
|
28
|
+
if const_defined?("Proxy", false)
|
|
29
29
|
Proxy.prepend(self::Proxy)
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
-
if const_defined?(
|
|
32
|
+
if const_defined?("Builder", false)
|
|
33
33
|
Builder.prepend(self::Builder)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
-
if const_defined?(
|
|
36
|
+
if const_defined?("Contracts", false)
|
|
37
37
|
Builder.prepend(Base::Expander.new(self))
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
-
if const_defined?(
|
|
40
|
+
if const_defined?("SubscriptionGroup", false)
|
|
41
41
|
SubscriptionGroup.prepend(self::SubscriptionGroup)
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
-
if const_defined?(
|
|
44
|
+
if const_defined?("SubscriptionGroupsBuilder", false)
|
|
45
45
|
SubscriptionGroupsBuilder.prepend(self::SubscriptionGroupsBuilder)
|
|
46
46
|
end
|
|
47
47
|
end
|
|
@@ -10,8 +10,8 @@ module Karafka
|
|
|
10
10
|
class Topic < Karafka::Contracts::Base
|
|
11
11
|
configure do |config|
|
|
12
12
|
config.error_messages = YAML.safe_load_file(
|
|
13
|
-
File.join(Karafka.gem_root,
|
|
14
|
-
).fetch(
|
|
13
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
14
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
nested :dead_letter_queue do
|
|
@@ -10,8 +10,8 @@ module Karafka
|
|
|
10
10
|
class Topic < Karafka::Contracts::Base
|
|
11
11
|
configure do |config|
|
|
12
12
|
config.error_messages = YAML.safe_load_file(
|
|
13
|
-
File.join(Karafka.gem_root,
|
|
14
|
-
).fetch(
|
|
13
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
14
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
nested :declaratives do
|
|
@@ -10,8 +10,8 @@ module Karafka
|
|
|
10
10
|
class Topic < Karafka::Contracts::Base
|
|
11
11
|
configure do |config|
|
|
12
12
|
config.error_messages = YAML.safe_load_file(
|
|
13
|
-
File.join(Karafka.gem_root,
|
|
14
|
-
).fetch(
|
|
13
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
14
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
nested :deserializers do
|
|
@@ -10,8 +10,8 @@ module Karafka
|
|
|
10
10
|
class Topic < Karafka::Contracts::Base
|
|
11
11
|
configure do |config|
|
|
12
12
|
config.error_messages = YAML.safe_load_file(
|
|
13
|
-
File.join(Karafka.gem_root,
|
|
14
|
-
).fetch(
|
|
13
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
14
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
nested :eofed do
|
|
@@ -25,7 +25,7 @@ module Karafka
|
|
|
25
25
|
|
|
26
26
|
next unless eofed[:active]
|
|
27
27
|
|
|
28
|
-
next if data[:kafka][:
|
|
28
|
+
next if data[:kafka][:"enable.partition.eof"]
|
|
29
29
|
|
|
30
30
|
[[%i[eofed kafka], :enable]]
|
|
31
31
|
end
|
|
@@ -10,8 +10,8 @@ module Karafka
|
|
|
10
10
|
class Topic < Karafka::Contracts::Base
|
|
11
11
|
configure do |config|
|
|
12
12
|
config.error_messages = YAML.safe_load_file(
|
|
13
|
-
File.join(Karafka.gem_root,
|
|
14
|
-
).fetch(
|
|
13
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
14
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
nested :inline_insights do
|
|
@@ -12,17 +12,17 @@ module Karafka
|
|
|
12
12
|
#
|
|
13
13
|
# @param _config [Karafka::Core::Configurable::Node] app config
|
|
14
14
|
def post_setup(_config)
|
|
15
|
-
Karafka::App.monitor.subscribe(
|
|
15
|
+
Karafka::App.monitor.subscribe("app.running") do
|
|
16
16
|
# Do not activate tracking of statistics if none of our active topics uses it
|
|
17
17
|
# This prevents us from tracking metrics when user just runs a subset of topics
|
|
18
18
|
# in a given process and none of those actually utilizes this feature
|
|
19
19
|
next unless Karafka::App
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
20
|
+
.subscription_groups
|
|
21
|
+
.values
|
|
22
|
+
.flat_map(&:itself)
|
|
23
|
+
.flat_map(&:topics)
|
|
24
|
+
.flat_map(&:to_a)
|
|
25
|
+
.any?(&:inline_insights?)
|
|
26
26
|
|
|
27
27
|
# Initialize the tracker prior to becoming multi-threaded
|
|
28
28
|
Karafka::Processing::InlineInsights::Tracker.instance
|
|
@@ -10,8 +10,8 @@ module Karafka
|
|
|
10
10
|
class Topic < Karafka::Contracts::Base
|
|
11
11
|
configure do |config|
|
|
12
12
|
config.error_messages = YAML.safe_load_file(
|
|
13
|
-
File.join(Karafka.gem_root,
|
|
14
|
-
).fetch(
|
|
13
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
14
|
+
).fetch("en").fetch("validations").fetch("routing").fetch("topic")
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
nested :manual_offset_management do
|
|
@@ -58,7 +58,7 @@ module Karafka
|
|
|
58
58
|
|
|
59
59
|
# @return [String] consumer group id
|
|
60
60
|
def consumer_group_id
|
|
61
|
-
kafka[:
|
|
61
|
+
kafka[:"group.id"]
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
# @return [Integer] max messages fetched in a single go
|
|
@@ -107,7 +107,7 @@ module Karafka
|
|
|
107
107
|
# node identifier. This refreshes this if needed when in swarm.
|
|
108
108
|
def refresh
|
|
109
109
|
return unless node
|
|
110
|
-
return unless kafka.key?(:
|
|
110
|
+
return unless kafka.key?(:"group.instance.id")
|
|
111
111
|
|
|
112
112
|
@kafka = build_kafka
|
|
113
113
|
end
|
|
@@ -124,11 +124,11 @@ module Karafka
|
|
|
124
124
|
inject_group_instance_id(kafka)
|
|
125
125
|
inject_client_id(kafka)
|
|
126
126
|
|
|
127
|
-
kafka[:
|
|
128
|
-
kafka[:
|
|
127
|
+
kafka[:"group.id"] ||= @consumer_group.id
|
|
128
|
+
kafka[:"auto.offset.reset"] ||= @topics.first.initial_offset
|
|
129
129
|
# Karafka manages the offsets based on the processing state, thus we do not rely on the
|
|
130
130
|
# rdkafka offset auto-storing
|
|
131
|
-
kafka[:
|
|
131
|
+
kafka[:"enable.auto.offset.store"] = false
|
|
132
132
|
kafka.freeze
|
|
133
133
|
kafka
|
|
134
134
|
end
|
|
@@ -146,9 +146,9 @@ module Karafka
|
|
|
146
146
|
def inject_client_id(kafka)
|
|
147
147
|
# If client id is set directly on librdkafka level, we do nothing and just go with what
|
|
148
148
|
# end user has configured
|
|
149
|
-
return if kafka.key?(:
|
|
149
|
+
return if kafka.key?(:"client.id")
|
|
150
150
|
|
|
151
|
-
kafka[:
|
|
151
|
+
kafka[:"client.id"] = client_id
|
|
152
152
|
end
|
|
153
153
|
|
|
154
154
|
# If we use static group memberships, there can be a case, where same instance id would
|
|
@@ -161,7 +161,7 @@ module Karafka
|
|
|
161
161
|
# affecting the instance id and causing conflicts
|
|
162
162
|
# @param kafka [Hash] kafka level config
|
|
163
163
|
def inject_group_instance_id(kafka)
|
|
164
|
-
group_instance_prefix = kafka.fetch(:
|
|
164
|
+
group_instance_prefix = kafka.fetch(:"group.instance.id", false)
|
|
165
165
|
|
|
166
166
|
# If group instance id was not even configured, do nothing
|
|
167
167
|
return unless group_instance_prefix
|
|
@@ -170,7 +170,7 @@ module Karafka
|
|
|
170
170
|
# have different instances ids but they are reproducible
|
|
171
171
|
components = [group_instance_prefix, node ? node.id : nil, @position]
|
|
172
172
|
|
|
173
|
-
kafka[:
|
|
173
|
+
kafka[:"group.instance.id"] = components.compact.join("_")
|
|
174
174
|
end
|
|
175
175
|
end
|
|
176
176
|
end
|
data/lib/karafka/runner.rb
CHANGED
|
@@ -59,12 +59,12 @@ module Karafka
|
|
|
59
59
|
workers.each(&:join)
|
|
60
60
|
# If anything crashes here, we need to raise the error and crush the runner because it means
|
|
61
61
|
# that something terrible happened
|
|
62
|
-
rescue
|
|
62
|
+
rescue => e
|
|
63
63
|
Karafka.monitor.instrument(
|
|
64
|
-
|
|
64
|
+
"error.occurred",
|
|
65
65
|
caller: self,
|
|
66
66
|
error: e,
|
|
67
|
-
type:
|
|
67
|
+
type: "runner.call.error"
|
|
68
68
|
)
|
|
69
69
|
Karafka::App.stop!
|
|
70
70
|
raise e
|
data/lib/karafka/server.rb
CHANGED
|
@@ -127,10 +127,10 @@ module Karafka
|
|
|
127
127
|
raise Errors::ForcefulShutdownError
|
|
128
128
|
rescue Errors::ForcefulShutdownError => e
|
|
129
129
|
Karafka.monitor.instrument(
|
|
130
|
-
|
|
130
|
+
"error.occurred",
|
|
131
131
|
caller: self,
|
|
132
132
|
error: e,
|
|
133
|
-
type:
|
|
133
|
+
type: "app.stopping.error"
|
|
134
134
|
)
|
|
135
135
|
|
|
136
136
|
# We're done waiting, lets kill them!
|
|
@@ -144,13 +144,13 @@ module Karafka
|
|
|
144
144
|
# indefinitely even with risk of VM crash as this is a last resort.
|
|
145
145
|
Thread.new do
|
|
146
146
|
listeners.each(&:shutdown)
|
|
147
|
-
rescue
|
|
147
|
+
rescue => e
|
|
148
148
|
# If anything wrong happened during shutdown, we also want to record it
|
|
149
149
|
Karafka.monitor.instrument(
|
|
150
|
-
|
|
150
|
+
"error.occurred",
|
|
151
151
|
caller: self,
|
|
152
152
|
error: e,
|
|
153
|
-
type:
|
|
153
|
+
type: "app.forceful_stopping.error"
|
|
154
154
|
)
|
|
155
155
|
end.join(forceful_shutdown_wait / 1_000.0)
|
|
156
156
|
|