karafka 2.5.4 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/LICENSE-COMM +8 -2
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +128 -81
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +84 -70
- data/lib/karafka/admin/consumer_groups.rb +377 -330
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +304 -280
- data/lib/karafka/admin/topics.rb +232 -186
- data/lib/karafka/admin.rb +276 -117
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +53 -53
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +28 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/consumer.rb +19 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +19 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +21 -4
- data/lib/karafka/pro/base_consumer.rb +19 -2
- data/lib/karafka/pro/cleaner/errors.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/message.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +29 -2
- data/lib/karafka/pro/cleaner/messages/metadata.rb +19 -2
- data/lib/karafka/pro/cleaner.rb +19 -2
- data/lib/karafka/pro/cli/contracts/server.rb +31 -14
- data/lib/karafka/pro/cli/parallel_segments/base.rb +23 -6
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +24 -7
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +22 -5
- data/lib/karafka/pro/cli/parallel_segments.rb +26 -9
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +33 -16
- data/lib/karafka/pro/connection/multiplexing/listener.rb +19 -2
- data/lib/karafka/pro/contracts/base.rb +19 -2
- data/lib/karafka/pro/encryption/cipher.rb +19 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +21 -4
- data/lib/karafka/pro/encryption/errors.rb +19 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +21 -4
- data/lib/karafka/pro/encryption/messages/parser.rb +21 -4
- data/lib/karafka/pro/encryption/setup/config.rb +21 -4
- data/lib/karafka/pro/encryption.rb +19 -2
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +19 -2
- data/lib/karafka/pro/iterator/expander.rb +19 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +21 -4
- data/lib/karafka/pro/iterator.rb +20 -3
- data/lib/karafka/pro/loader.rb +21 -3
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +20 -3
- data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +19 -2
- data/lib/karafka/pro/processing/collapser.rb +19 -2
- data/lib/karafka/pro/processing/coordinator.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +26 -9
- data/lib/karafka/pro/processing/executor.rb +19 -2
- data/lib/karafka/pro/processing/expansions_selector.rb +19 -2
- data/lib/karafka/pro/processing/filters/base.rb +20 -3
- data/lib/karafka/pro/processing/filters/delayer.rb +21 -4
- data/lib/karafka/pro/processing/filters/expirer.rb +19 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +20 -3
- data/lib/karafka/pro/processing/filters/throttler.rb +19 -2
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +19 -2
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs_builder.rb +19 -2
- data/lib/karafka/pro/processing/jobs_queue.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +20 -3
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/listener.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +25 -8
- data/lib/karafka/pro/processing/parallel_segments/filters/default.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb +19 -2
- data/lib/karafka/pro/processing/partitioner.rb +22 -5
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +25 -7
- data/lib/karafka/pro/processing/piping/consumer.rb +26 -9
- data/lib/karafka/pro/processing/schedulers/base.rb +24 -7
- data/lib/karafka/pro/processing/schedulers/default.rb +24 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +21 -4
- data/lib/karafka/pro/processing/strategies/aj/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/base.rb +19 -2
- data/lib/karafka/pro/processing/strategies/default.rb +41 -24
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +26 -9
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/ftr/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/vp/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies.rb +19 -2
- data/lib/karafka/pro/processing/strategy_selector.rb +19 -2
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/base.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/errors.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/executor.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +20 -3
- data/lib/karafka/pro/recurring_tasks/matcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/schedule.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +24 -7
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +22 -5
- data/lib/karafka/pro/recurring_tasks/task.rb +23 -6
- data/lib/karafka/pro/recurring_tasks.rb +23 -6
- data/lib/karafka/pro/routing/features/active_job/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/active_job.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator.rb +19 -2
- data/lib/karafka/pro/routing/features/base.rb +19 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/dead_letter_queue.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/config.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/direct_assignments.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/config.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/config.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/config.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/inline_insights/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/long_running_job.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +24 -7
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/non_blocking_job.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/config.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/offset_metadata.rb +20 -3
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +24 -7
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/parallel_segments/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/config.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +22 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/detector.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/patterns/topics.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/pausing/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/periodic_job.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +26 -9
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +32 -15
- data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/config.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm.rb +20 -3
- data/lib/karafka/pro/routing/features/throttling/config.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/throttling/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +26 -9
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +29 -12
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/day.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +24 -7
- data/lib/karafka/pro/scheduled_messages/errors.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/proxy.rb +27 -10
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/serializer.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/state.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/tracker.rb +21 -4
- data/lib/karafka/pro/scheduled_messages.rb +21 -4
- data/lib/karafka/pro/swarm/liveness_listener.rb +21 -4
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +5 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +7 -7
- data/lib/karafka/swarm/supervisor.rb +6 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -4
|
@@ -47,7 +47,7 @@ module Karafka
|
|
|
47
47
|
#
|
|
48
48
|
# @param event [Karafka::Core::Monitoring::Event] event details including payload
|
|
49
49
|
def on_worker_process(event)
|
|
50
|
-
current_span = client.trace(
|
|
50
|
+
current_span = client.trace("karafka.consumer", service: service_name)
|
|
51
51
|
push_tags
|
|
52
52
|
|
|
53
53
|
job = event[:job]
|
|
@@ -69,7 +69,7 @@ module Karafka
|
|
|
69
69
|
|
|
70
70
|
job = event[:job]
|
|
71
71
|
time = event[:time]
|
|
72
|
-
job_type = job.class.to_s.split(
|
|
72
|
+
job_type = job.class.to_s.split("::").last
|
|
73
73
|
consumer = job.executor.topic.consumer
|
|
74
74
|
topic = job.executor.topic.name
|
|
75
75
|
|
|
@@ -90,55 +90,55 @@ module Karafka
|
|
|
90
90
|
client.active_span&.set_error(error)
|
|
91
91
|
|
|
92
92
|
case event[:type]
|
|
93
|
-
when
|
|
93
|
+
when "consumer.initialized.error"
|
|
94
94
|
error "Consumer initialized error: #{error}"
|
|
95
|
-
when
|
|
95
|
+
when "consumer.wrap.error"
|
|
96
96
|
error "Consumer wrap failed due to an error: #{error}"
|
|
97
|
-
when
|
|
97
|
+
when "consumer.consume.error"
|
|
98
98
|
error "Consumer consuming error: #{error}"
|
|
99
|
-
when
|
|
99
|
+
when "consumer.revoked.error"
|
|
100
100
|
error "Consumer on revoked failed due to an error: #{error}"
|
|
101
|
-
when
|
|
101
|
+
when "consumer.idle.error"
|
|
102
102
|
error "Consumer idle failed due to an error: #{error}"
|
|
103
|
-
when
|
|
103
|
+
when "consumer.shutdown.error"
|
|
104
104
|
error "Consumer on shutdown failed due to an error: #{error}"
|
|
105
|
-
when
|
|
105
|
+
when "consumer.tick.error"
|
|
106
106
|
error "Consumer on tick failed due to an error: #{error}"
|
|
107
|
-
when
|
|
107
|
+
when "consumer.eofed.error"
|
|
108
108
|
error "Consumer on eofed failed due to an error: #{error}"
|
|
109
|
-
when
|
|
109
|
+
when "consumer.after_consume.error"
|
|
110
110
|
error "Consumer on after_consume failed due to an error: #{error}"
|
|
111
|
-
when
|
|
111
|
+
when "worker.process.error"
|
|
112
112
|
fatal "Worker processing failed due to an error: #{error}"
|
|
113
|
-
when
|
|
113
|
+
when "connection.listener.fetch_loop.error"
|
|
114
114
|
error "Listener fetch loop error: #{error}"
|
|
115
|
-
when
|
|
115
|
+
when "swarm.supervisor.error"
|
|
116
116
|
fatal "Swarm supervisor crashed due to an error: #{error}"
|
|
117
|
-
when
|
|
117
|
+
when "runner.call.error"
|
|
118
118
|
fatal "Runner crashed due to an error: #{error}"
|
|
119
|
-
when
|
|
120
|
-
error
|
|
121
|
-
when
|
|
119
|
+
when "app.stopping.error"
|
|
120
|
+
error "Forceful Karafka server stop"
|
|
121
|
+
when "app.forceful_stopping.error"
|
|
122
122
|
error "Forceful shutdown error occurred: #{error}"
|
|
123
|
-
when
|
|
123
|
+
when "librdkafka.error"
|
|
124
124
|
error "librdkafka internal error occurred: #{error}"
|
|
125
|
-
when
|
|
125
|
+
when "callbacks.statistics.error"
|
|
126
126
|
error "callbacks.statistics processing failed due to an error: #{error}"
|
|
127
|
-
when
|
|
127
|
+
when "callbacks.error.error"
|
|
128
128
|
error "callbacks.error processing failed due to an error: #{error}"
|
|
129
129
|
# Those will only occur when retries in the client fail and when they did not stop
|
|
130
130
|
# after back-offs
|
|
131
|
-
when
|
|
131
|
+
when "connection.client.poll.error"
|
|
132
132
|
error "Data polling error occurred: #{error}"
|
|
133
|
-
when
|
|
133
|
+
when "connection.client.rebalance_callback.error"
|
|
134
134
|
error "Rebalance callback error occurred: #{error}"
|
|
135
|
-
when
|
|
135
|
+
when "connection.client.unsubscribe.error"
|
|
136
136
|
error "Client unsubscribe error occurred: #{error}"
|
|
137
|
-
when
|
|
137
|
+
when "parallel_segments.reducer.error"
|
|
138
138
|
error "Parallel segments reducer error occurred: #{error}"
|
|
139
|
-
when
|
|
139
|
+
when "parallel_segments.partitioner.error"
|
|
140
140
|
error "Parallel segments partitioner error occurred: #{error}"
|
|
141
|
-
when
|
|
141
|
+
when "virtual_partitions.partitioner.error"
|
|
142
142
|
error "Virtual partitions partitioner error occurred: #{error}"
|
|
143
143
|
else
|
|
144
144
|
error "#{event[:type]} error occurred: #{error}"
|
|
@@ -180,7 +180,7 @@ module Karafka
|
|
|
180
180
|
# the assignment race condition is irrelevant here since the same value will be
|
|
181
181
|
# assigned.
|
|
182
182
|
def fetch_job_type(job_class)
|
|
183
|
-
@job_types_cache[job_class] ||= job_class.to_s.split(
|
|
183
|
+
@job_types_cache[job_class] ||= job_class.to_s.split("::").last
|
|
184
184
|
end
|
|
185
185
|
end
|
|
186
186
|
end
|
|
@@ -22,7 +22,7 @@ module Karafka
|
|
|
22
22
|
RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
|
|
23
23
|
|
|
24
24
|
# Namespace under which the DD metrics should be published
|
|
25
|
-
setting :namespace, default:
|
|
25
|
+
setting :namespace, default: "karafka"
|
|
26
26
|
|
|
27
27
|
# Datadog client that we should use to publish the metrics
|
|
28
28
|
setting :client
|
|
@@ -37,22 +37,22 @@ module Karafka
|
|
|
37
37
|
# Note, that the once with `_d` come from Karafka, not rdkafka or Kafka
|
|
38
38
|
setting :rd_kafka_metrics, default: [
|
|
39
39
|
# Client metrics
|
|
40
|
-
RdKafkaMetric.new(:count, :root,
|
|
41
|
-
RdKafkaMetric.new(:count, :root,
|
|
40
|
+
RdKafkaMetric.new(:count, :root, "messages.consumed", "rxmsgs_d"),
|
|
41
|
+
RdKafkaMetric.new(:count, :root, "messages.consumed.bytes", "rxmsg_bytes"),
|
|
42
42
|
|
|
43
43
|
# Broker metrics
|
|
44
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
45
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
46
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
47
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
48
|
-
RdKafkaMetric.new(:count, :brokers,
|
|
49
|
-
RdKafkaMetric.new(:gauge, :brokers,
|
|
50
|
-
RdKafkaMetric.new(:gauge, :brokers,
|
|
51
|
-
RdKafkaMetric.new(:gauge, :brokers,
|
|
44
|
+
RdKafkaMetric.new(:count, :brokers, "consume.attempts", "txretries_d"),
|
|
45
|
+
RdKafkaMetric.new(:count, :brokers, "consume.errors", "txerrs_d"),
|
|
46
|
+
RdKafkaMetric.new(:count, :brokers, "receive.errors", "rxerrs_d"),
|
|
47
|
+
RdKafkaMetric.new(:count, :brokers, "connection.connects", "connects_d"),
|
|
48
|
+
RdKafkaMetric.new(:count, :brokers, "connection.disconnects", "disconnects_d"),
|
|
49
|
+
RdKafkaMetric.new(:gauge, :brokers, "network.latency.avg", %w[rtt avg]),
|
|
50
|
+
RdKafkaMetric.new(:gauge, :brokers, "network.latency.p95", %w[rtt p95]),
|
|
51
|
+
RdKafkaMetric.new(:gauge, :brokers, "network.latency.p99", %w[rtt p99]),
|
|
52
52
|
|
|
53
53
|
# Topics metrics
|
|
54
|
-
RdKafkaMetric.new(:gauge, :topics,
|
|
55
|
-
RdKafkaMetric.new(:gauge, :topics,
|
|
54
|
+
RdKafkaMetric.new(:gauge, :topics, "consumer.lags", "consumer_lag_stored"),
|
|
55
|
+
RdKafkaMetric.new(:gauge, :topics, "consumer.lags_delta", "consumer_lag_stored_d")
|
|
56
56
|
].freeze
|
|
57
57
|
|
|
58
58
|
# Whether histogram metrics should be sent as distributions or histograms.
|
|
@@ -101,7 +101,7 @@ module Karafka
|
|
|
101
101
|
tags.concat(consumer_tags(event.payload[:caller]))
|
|
102
102
|
end
|
|
103
103
|
|
|
104
|
-
count(
|
|
104
|
+
count("error_occurred", 1, tags: tags)
|
|
105
105
|
end
|
|
106
106
|
|
|
107
107
|
# Reports how many messages we've polled and how much time did we spend on it
|
|
@@ -116,8 +116,8 @@ module Karafka
|
|
|
116
116
|
tags = ["consumer_group:#{consumer_group_id}"]
|
|
117
117
|
tags.concat(default_tags)
|
|
118
118
|
|
|
119
|
-
histogram(
|
|
120
|
-
histogram(
|
|
119
|
+
histogram("listener.polling.time_taken", time_taken, tags: tags)
|
|
120
|
+
histogram("listener.polling.messages", messages_count, tags: tags)
|
|
121
121
|
end
|
|
122
122
|
|
|
123
123
|
# Here we report majority of things related to processing as we have access to the
|
|
@@ -131,13 +131,13 @@ module Karafka
|
|
|
131
131
|
tags = consumer_tags(consumer)
|
|
132
132
|
tags.concat(default_tags)
|
|
133
133
|
|
|
134
|
-
count(
|
|
135
|
-
count(
|
|
136
|
-
gauge(
|
|
137
|
-
histogram(
|
|
138
|
-
histogram(
|
|
139
|
-
histogram(
|
|
140
|
-
histogram(
|
|
134
|
+
count("consumer.messages", messages.size, tags: tags)
|
|
135
|
+
count("consumer.batches", 1, tags: tags)
|
|
136
|
+
gauge("consumer.offset", metadata.last_offset, tags: tags)
|
|
137
|
+
histogram("consumer.consumed.time_taken", event[:time], tags: tags)
|
|
138
|
+
histogram("consumer.batch_size", messages.size, tags: tags)
|
|
139
|
+
histogram("consumer.processing_lag", metadata.processing_lag, tags: tags)
|
|
140
|
+
histogram("consumer.consumption_lag", metadata.consumption_lag, tags: tags)
|
|
141
141
|
end
|
|
142
142
|
|
|
143
143
|
{
|
|
@@ -164,9 +164,9 @@ module Karafka
|
|
|
164
164
|
jq_stats = event[:jobs_queue].statistics
|
|
165
165
|
|
|
166
166
|
tags = default_tags
|
|
167
|
-
gauge(
|
|
168
|
-
histogram(
|
|
169
|
-
histogram(
|
|
167
|
+
gauge("worker.total_threads", Karafka::App.config.concurrency, tags: tags)
|
|
168
|
+
histogram("worker.processing", jq_stats[:busy], tags: tags)
|
|
169
|
+
histogram("worker.enqueued_jobs", jq_stats[:enqueued], tags: tags)
|
|
170
170
|
end
|
|
171
171
|
|
|
172
172
|
# We report this metric before and after processing for higher accuracy
|
|
@@ -175,7 +175,7 @@ module Karafka
|
|
|
175
175
|
def on_worker_processed(event)
|
|
176
176
|
jq_stats = event[:jobs_queue].statistics
|
|
177
177
|
|
|
178
|
-
histogram(
|
|
178
|
+
histogram("worker.processing", jq_stats[:busy], tags: default_tags)
|
|
179
179
|
end
|
|
180
180
|
|
|
181
181
|
private
|
|
@@ -213,7 +213,7 @@ module Karafka
|
|
|
213
213
|
else
|
|
214
214
|
raise(
|
|
215
215
|
ArgumentError,
|
|
216
|
-
|
|
216
|
+
"distribution_mode setting value must be either :histogram or :distribution"
|
|
217
217
|
)
|
|
218
218
|
end
|
|
219
219
|
end
|
|
@@ -239,13 +239,13 @@ module Karafka
|
|
|
239
239
|
tags: base_tags
|
|
240
240
|
)
|
|
241
241
|
when :brokers
|
|
242
|
-
statistics.fetch(
|
|
242
|
+
statistics.fetch("brokers").each_value do |broker_statistics|
|
|
243
243
|
# Skip bootstrap nodes
|
|
244
244
|
# Bootstrap nodes have nodeid -1, other nodes have positive
|
|
245
245
|
# node ids
|
|
246
|
-
next if broker_statistics[
|
|
246
|
+
next if broker_statistics["nodeid"] == -1
|
|
247
247
|
|
|
248
|
-
tags = ["broker:#{broker_statistics[
|
|
248
|
+
tags = ["broker:#{broker_statistics["nodename"]}"]
|
|
249
249
|
tags.concat(base_tags)
|
|
250
250
|
|
|
251
251
|
public_send(
|
|
@@ -256,16 +256,16 @@ module Karafka
|
|
|
256
256
|
)
|
|
257
257
|
end
|
|
258
258
|
when :topics
|
|
259
|
-
statistics.fetch(
|
|
260
|
-
topic_values[
|
|
261
|
-
next if partition_name ==
|
|
259
|
+
statistics.fetch("topics").each do |topic_name, topic_values|
|
|
260
|
+
topic_values["partitions"].each do |partition_name, partition_statistics|
|
|
261
|
+
next if partition_name == "-1"
|
|
262
262
|
# Skip until lag info is available
|
|
263
|
-
next if partition_statistics[
|
|
264
|
-
next if partition_statistics[
|
|
263
|
+
next if partition_statistics["consumer_lag"] == -1
|
|
264
|
+
next if partition_statistics["consumer_lag_stored"] == -1
|
|
265
265
|
|
|
266
266
|
# Skip if we do not own the fetch assignment
|
|
267
|
-
next if partition_statistics[
|
|
268
|
-
next if partition_statistics[
|
|
267
|
+
next if partition_statistics["fetch_state"] == "stopped"
|
|
268
|
+
next if partition_statistics["fetch_state"] == "none"
|
|
269
269
|
|
|
270
270
|
tags = ["topic:#{topic_name}", "partition:#{partition_name}"]
|
|
271
271
|
tags.concat(base_tags)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "socket"
|
|
4
4
|
|
|
5
5
|
module Karafka
|
|
6
6
|
module Instrumentation
|
|
@@ -13,10 +13,10 @@ module Karafka
|
|
|
13
13
|
include Karafka::Core::Helpers::Time
|
|
14
14
|
|
|
15
15
|
# All good with Karafka
|
|
16
|
-
OK_CODE =
|
|
16
|
+
OK_CODE = "200 OK"
|
|
17
17
|
|
|
18
18
|
# Some timeouts, fail
|
|
19
|
-
FAIL_CODE =
|
|
19
|
+
FAIL_CODE = "500 Internal Server Error"
|
|
20
20
|
|
|
21
21
|
private_constant :OK_CODE, :FAIL_CODE
|
|
22
22
|
|
|
@@ -32,7 +32,7 @@ module Karafka
|
|
|
32
32
|
|
|
33
33
|
# @return [Boolean] true if all good, false if we should tell k8s to kill this process
|
|
34
34
|
def healthy?
|
|
35
|
-
raise NotImplementedError,
|
|
35
|
+
raise NotImplementedError, "Implement in a subclass"
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
private
|
|
@@ -58,7 +58,7 @@ module Karafka
|
|
|
58
58
|
# @return [Hash] hash that will be the response body
|
|
59
59
|
def status_body
|
|
60
60
|
{
|
|
61
|
-
status: healthy? ?
|
|
61
|
+
status: healthy? ? "healthy" : "unhealthy",
|
|
62
62
|
timestamp: Time.now.to_i,
|
|
63
63
|
port: @port,
|
|
64
64
|
process_id: ::Process.pid
|
data/lib/karafka/licenser.rb
CHANGED
|
@@ -4,15 +4,25 @@ module Karafka
|
|
|
4
4
|
# Checks the license presence for pro and loads pro components when needed (if any)
|
|
5
5
|
class Licenser
|
|
6
6
|
# Location in the gem where we store the public key
|
|
7
|
-
PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root,
|
|
7
|
+
PUBLIC_KEY_LOCATION = File.join(Karafka.gem_root, "certs", "karafka-pro.pem")
|
|
8
|
+
|
|
9
|
+
# Location of file containing expired/revoked license checksums
|
|
10
|
+
EXPIRED_CHECKSUMS_LOCATION = File.join(Karafka.gem_root, "certs", "expired.txt")
|
|
8
11
|
|
|
9
12
|
private_constant :PUBLIC_KEY_LOCATION
|
|
10
13
|
|
|
11
14
|
class << self
|
|
12
15
|
# Tries to load the license and yields if successful
|
|
13
16
|
def detect
|
|
14
|
-
# If
|
|
15
|
-
|
|
17
|
+
# If license module is already fully defined, don't touch it
|
|
18
|
+
# This allows users to define their own License module for testing/custom setups
|
|
19
|
+
unless license_fully_defined?
|
|
20
|
+
# Try safe approach first (no code execution)
|
|
21
|
+
loaded = safe_load_license || fallback_require_license
|
|
22
|
+
|
|
23
|
+
# If neither method succeeded, return false
|
|
24
|
+
return false unless loaded
|
|
25
|
+
end
|
|
16
26
|
|
|
17
27
|
yield
|
|
18
28
|
|
|
@@ -26,7 +36,7 @@ module Karafka
|
|
|
26
36
|
# @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
|
|
27
37
|
def prepare_and_verify(license_config)
|
|
28
38
|
# If license is not loaded, nothing to do
|
|
29
|
-
return unless const_defined?(
|
|
39
|
+
return unless const_defined?("::Karafka::License")
|
|
30
40
|
|
|
31
41
|
prepare(license_config)
|
|
32
42
|
verify(license_config)
|
|
@@ -34,6 +44,65 @@ module Karafka
|
|
|
34
44
|
|
|
35
45
|
private
|
|
36
46
|
|
|
47
|
+
# Check if License module and required methods are already defined
|
|
48
|
+
# @return [Boolean]
|
|
49
|
+
def license_fully_defined?
|
|
50
|
+
return false unless const_defined?("::Karafka::License")
|
|
51
|
+
|
|
52
|
+
# Check if the required methods exist
|
|
53
|
+
::Karafka::License.respond_to?(:token) &&
|
|
54
|
+
::Karafka::License.respond_to?(:version)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Attempt to safely load license without executing gem code
|
|
58
|
+
# @return [Boolean] true if successful, false otherwise
|
|
59
|
+
def safe_load_license
|
|
60
|
+
# First verify the gem exists and files are accessible
|
|
61
|
+
spec = Gem::Specification.find_by_name("karafka-license")
|
|
62
|
+
license_path = File.join(spec.gem_dir, "lib", "license.txt")
|
|
63
|
+
version_path = File.join(spec.gem_dir, "lib", "version.txt")
|
|
64
|
+
|
|
65
|
+
return false unless File.exist?(license_path)
|
|
66
|
+
|
|
67
|
+
# Read license data to ensure we can successfully load before removing any constants
|
|
68
|
+
license_token = File.read(license_path)
|
|
69
|
+
version_content = File.read(version_path)
|
|
70
|
+
|
|
71
|
+
# Only after confirming we have valid data, remove and replace the License constant
|
|
72
|
+
# This ensures we always use the actual license from the gem, overwriting any stale
|
|
73
|
+
# or incomplete License module that may have been defined elsewhere
|
|
74
|
+
::Karafka.send(:remove_const, :License) if const_defined?("::Karafka::License")
|
|
75
|
+
|
|
76
|
+
::Karafka.const_set(:License, Module.new)
|
|
77
|
+
::Karafka::License.define_singleton_method(:token) { license_token }
|
|
78
|
+
::Karafka::License.define_singleton_method(:version) { version_content }
|
|
79
|
+
|
|
80
|
+
true
|
|
81
|
+
rescue Gem::MissingSpecError, Errno::ENOENT
|
|
82
|
+
# Gem not found or files don't exist - don't touch existing License constant
|
|
83
|
+
false
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Fallback to traditional require if safe method fails
|
|
87
|
+
# @return [Boolean] true if successful, false otherwise
|
|
88
|
+
def fallback_require_license
|
|
89
|
+
# First check if we can actually load the gem before removing any constants
|
|
90
|
+
# Try to find the gem spec to see if it exists
|
|
91
|
+
begin
|
|
92
|
+
Gem::Specification.find_by_name("karafka-license")
|
|
93
|
+
rescue Gem::MissingSpecError
|
|
94
|
+
return false
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Gem exists, so we can safely remove and reload the License constant
|
|
98
|
+
::Karafka.send(:remove_const, :License) if const_defined?("::Karafka::License")
|
|
99
|
+
|
|
100
|
+
require("karafka-license")
|
|
101
|
+
true
|
|
102
|
+
rescue LoadError
|
|
103
|
+
false
|
|
104
|
+
end
|
|
105
|
+
|
|
37
106
|
# @param license_config [Karafka::Core::Configurable::Node] config related to the licensing
|
|
38
107
|
def prepare(license_config)
|
|
39
108
|
license_config.token = Karafka::License.token
|
|
@@ -45,8 +114,12 @@ module Karafka
|
|
|
45
114
|
public_key = OpenSSL::PKey::RSA.new(File.read(PUBLIC_KEY_LOCATION))
|
|
46
115
|
|
|
47
116
|
# We gsub and strip in case someone copy-pasted it as a multi line string
|
|
48
|
-
formatted_token = license_config.token.strip.delete("\n").delete(
|
|
49
|
-
|
|
117
|
+
formatted_token = license_config.token.strip.delete("\n").delete(" ")
|
|
118
|
+
|
|
119
|
+
# Check if the license has been revoked/expired before attempting decryption
|
|
120
|
+
raise_expired_license_token(license_config) if expired?(formatted_token)
|
|
121
|
+
|
|
122
|
+
decoded_token = formatted_token.unpack1("m") # decode from base64
|
|
50
123
|
|
|
51
124
|
begin
|
|
52
125
|
data = public_key.public_decrypt(decoded_token)
|
|
@@ -56,7 +129,26 @@ module Karafka
|
|
|
56
129
|
|
|
57
130
|
details = data ? JSON.parse(data) : raise_invalid_license_token(license_config)
|
|
58
131
|
|
|
59
|
-
license_config.entity = details.fetch(
|
|
132
|
+
license_config.entity = details.fetch("entity")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Checks if the license token has been expired/revoked
|
|
136
|
+
# @param formatted_token [String] formatted license token
|
|
137
|
+
# @return [Boolean] true if the license is in the expired list
|
|
138
|
+
def expired?(formatted_token)
|
|
139
|
+
token_checksum = Digest::SHA256.hexdigest(formatted_token)
|
|
140
|
+
|
|
141
|
+
expired_checksums.include?(token_checksum)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Loads the list of expired license checksums
|
|
145
|
+
# @return [Set<String>] set of expired license checksums
|
|
146
|
+
def expired_checksums
|
|
147
|
+
File
|
|
148
|
+
.readlines(EXPIRED_CHECKSUMS_LOCATION)
|
|
149
|
+
.map(&:strip)
|
|
150
|
+
.reject { |line| line.empty? || line.start_with?("#") }
|
|
151
|
+
.to_set
|
|
60
152
|
end
|
|
61
153
|
|
|
62
154
|
# Raises an error with info, that used token is invalid
|
|
@@ -67,12 +159,27 @@ module Karafka
|
|
|
67
159
|
|
|
68
160
|
raise(
|
|
69
161
|
Errors::InvalidLicenseTokenError,
|
|
70
|
-
<<~MSG.tr("\n",
|
|
162
|
+
<<~MSG.tr("\n", " ")
|
|
71
163
|
License key you provided is invalid.
|
|
72
164
|
Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
|
|
73
165
|
MSG
|
|
74
166
|
)
|
|
75
167
|
end
|
|
168
|
+
|
|
169
|
+
# Raises an error with info, that used token has expired or been revoked
|
|
170
|
+
# @param license_config [Karafka::Core::Configurable::Node]
|
|
171
|
+
def raise_expired_license_token(license_config)
|
|
172
|
+
# We set it to false so `Karafka.pro?` method behaves as expected
|
|
173
|
+
license_config.token = false
|
|
174
|
+
|
|
175
|
+
raise(
|
|
176
|
+
Errors::ExpiredLicenseTokenError,
|
|
177
|
+
<<~MSG.tr("\n", " ")
|
|
178
|
+
License key you provided has expired or been revoked.
|
|
179
|
+
Please reach us at contact@karafka.io or visit https://karafka.io to obtain a valid one.
|
|
180
|
+
MSG
|
|
181
|
+
)
|
|
182
|
+
end
|
|
76
183
|
end
|
|
77
184
|
end
|
|
78
185
|
end
|
|
@@ -80,7 +80,7 @@ module Karafka
|
|
|
80
80
|
tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
|
|
81
81
|
opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
|
|
82
82
|
|
|
83
|
-
if RB.rd_kafka_rebalance_protocol(client_ptr) ==
|
|
83
|
+
if RB.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
|
|
84
84
|
pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
|
85
85
|
else
|
|
86
86
|
pr.on_eager_rebalance(client_ptr, code, partitions_ptr, tpl, opaque)
|
|
@@ -95,7 +95,7 @@ end
|
|
|
95
95
|
# At the moment there is no API in rdkafka-ruby to do so
|
|
96
96
|
Rdkafka::Bindings.send(
|
|
97
97
|
:remove_const,
|
|
98
|
-
|
|
98
|
+
"RebalanceCallback"
|
|
99
99
|
)
|
|
100
100
|
|
|
101
101
|
Rdkafka::Bindings.const_set(
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -11,8 +28,8 @@ module Karafka
|
|
|
11
28
|
class JobOptionsContract < Contracts::Base
|
|
12
29
|
configure do |config|
|
|
13
30
|
config.error_messages = YAML.safe_load_file(
|
|
14
|
-
File.join(Karafka.gem_root,
|
|
15
|
-
).fetch(
|
|
31
|
+
File.join(Karafka.gem_root, "config", "locales", "errors.yml")
|
|
32
|
+
).fetch("en").fetch("validations").fetch("job_options")
|
|
16
33
|
end
|
|
17
34
|
|
|
18
35
|
optional(:producer) { |val| val.nil? || val.respond_to?(:call) }
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|