karafka 2.5.4 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/LICENSE-COMM +8 -2
- data/README.md +2 -2
- data/certs/expired.txt +2 -0
- data/karafka.gemspec +23 -23
- data/lib/active_job/karafka.rb +2 -2
- data/lib/active_job/queue_adapters/karafka_adapter.rb +5 -5
- data/lib/karafka/active_job/consumer.rb +3 -3
- data/lib/karafka/active_job/current_attributes.rb +4 -4
- data/lib/karafka/active_job/job_options_contract.rb +2 -2
- data/lib/karafka/admin/acl.rb +128 -81
- data/lib/karafka/admin/configs/resource.rb +1 -1
- data/lib/karafka/admin/configs.rb +84 -70
- data/lib/karafka/admin/consumer_groups.rb +377 -330
- data/lib/karafka/admin/contracts/replication.rb +2 -2
- data/lib/karafka/admin/replication.rb +304 -280
- data/lib/karafka/admin/topics.rb +232 -186
- data/lib/karafka/admin.rb +276 -117
- data/lib/karafka/app.rb +3 -3
- data/lib/karafka/base_consumer.rb +34 -30
- data/lib/karafka/cli/base.rb +8 -8
- data/lib/karafka/cli/console.rb +1 -1
- data/lib/karafka/cli/contracts/server.rb +12 -12
- data/lib/karafka/cli/help.rb +2 -2
- data/lib/karafka/cli/info.rb +4 -4
- data/lib/karafka/cli/install.rb +11 -11
- data/lib/karafka/cli/server.rb +6 -6
- data/lib/karafka/cli/swarm.rb +1 -1
- data/lib/karafka/cli/topics/align.rb +4 -4
- data/lib/karafka/cli/topics/base.rb +5 -5
- data/lib/karafka/cli/topics/create.rb +2 -2
- data/lib/karafka/cli/topics/delete.rb +2 -2
- data/lib/karafka/cli/topics/help.rb +5 -1
- data/lib/karafka/cli/topics/plan.rb +16 -16
- data/lib/karafka/cli/topics/repartition.rb +3 -3
- data/lib/karafka/cli/topics.rb +22 -22
- data/lib/karafka/cli.rb +2 -2
- data/lib/karafka/connection/client.rb +17 -17
- data/lib/karafka/connection/listener.rb +6 -6
- data/lib/karafka/connection/mode.rb +1 -1
- data/lib/karafka/connection/proxy.rb +1 -1
- data/lib/karafka/connection/status.rb +2 -2
- data/lib/karafka/constraints.rb +3 -3
- data/lib/karafka/embedded.rb +3 -3
- data/lib/karafka/env.rb +4 -4
- data/lib/karafka/errors.rb +3 -0
- data/lib/karafka/execution_mode.rb +1 -1
- data/lib/karafka/helpers/config_importer.rb +2 -2
- data/lib/karafka/helpers/multi_delegator.rb +1 -1
- data/lib/karafka/instrumentation/assignments_tracker.rb +9 -9
- data/lib/karafka/instrumentation/callbacks/error.rb +5 -5
- data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb +4 -4
- data/lib/karafka/instrumentation/callbacks/rebalance.rb +6 -6
- data/lib/karafka/instrumentation/callbacks/statistics.rb +5 -5
- data/lib/karafka/instrumentation/logger.rb +7 -7
- data/lib/karafka/instrumentation/logger_listener.rb +53 -53
- data/lib/karafka/instrumentation/vendors/appsignal/base.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/client.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/errors_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb +36 -36
- data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb +28 -28
- data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb +38 -38
- data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb +5 -5
- data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb +1 -1
- data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb +1 -1
- data/lib/karafka/licenser.rb +115 -8
- data/lib/karafka/messages/messages.rb +1 -1
- data/lib/karafka/patches/rdkafka/bindings.rb +2 -2
- data/lib/karafka/pro/active_job/consumer.rb +19 -2
- data/lib/karafka/pro/active_job/dispatcher.rb +19 -2
- data/lib/karafka/pro/active_job/job_options_contract.rb +21 -4
- data/lib/karafka/pro/base_consumer.rb +19 -2
- data/lib/karafka/pro/cleaner/errors.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/message.rb +19 -2
- data/lib/karafka/pro/cleaner/messages/messages.rb +29 -2
- data/lib/karafka/pro/cleaner/messages/metadata.rb +19 -2
- data/lib/karafka/pro/cleaner.rb +19 -2
- data/lib/karafka/pro/cli/contracts/server.rb +31 -14
- data/lib/karafka/pro/cli/parallel_segments/base.rb +23 -6
- data/lib/karafka/pro/cli/parallel_segments/collapse.rb +24 -7
- data/lib/karafka/pro/cli/parallel_segments/distribute.rb +22 -5
- data/lib/karafka/pro/cli/parallel_segments.rb +26 -9
- data/lib/karafka/pro/cli/topics/health.rb +162 -0
- data/lib/karafka/pro/cli/topics.rb +52 -0
- data/lib/karafka/pro/connection/manager.rb +33 -16
- data/lib/karafka/pro/connection/multiplexing/listener.rb +19 -2
- data/lib/karafka/pro/contracts/base.rb +19 -2
- data/lib/karafka/pro/encryption/cipher.rb +19 -2
- data/lib/karafka/pro/encryption/contracts/config.rb +21 -4
- data/lib/karafka/pro/encryption/errors.rb +19 -2
- data/lib/karafka/pro/encryption/messages/middleware.rb +21 -4
- data/lib/karafka/pro/encryption/messages/parser.rb +21 -4
- data/lib/karafka/pro/encryption/setup/config.rb +21 -4
- data/lib/karafka/pro/encryption.rb +19 -2
- data/lib/karafka/pro/instrumentation/performance_tracker.rb +19 -2
- data/lib/karafka/pro/iterator/expander.rb +19 -2
- data/lib/karafka/pro/iterator/tpl_builder.rb +21 -4
- data/lib/karafka/pro/iterator.rb +20 -3
- data/lib/karafka/pro/loader.rb +21 -3
- data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb +20 -3
- data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb +19 -2
- data/lib/karafka/pro/processing/collapser.rb +19 -2
- data/lib/karafka/pro/processing/coordinator.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/errors_tracker.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/filters_applier.rb +19 -2
- data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb +26 -9
- data/lib/karafka/pro/processing/executor.rb +19 -2
- data/lib/karafka/pro/processing/expansions_selector.rb +19 -2
- data/lib/karafka/pro/processing/filters/base.rb +20 -3
- data/lib/karafka/pro/processing/filters/delayer.rb +21 -4
- data/lib/karafka/pro/processing/filters/expirer.rb +19 -2
- data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb +20 -3
- data/lib/karafka/pro/processing/filters/throttler.rb +19 -2
- data/lib/karafka/pro/processing/filters/virtual_limiter.rb +19 -2
- data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic.rb +19 -2
- data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb +19 -2
- data/lib/karafka/pro/processing/jobs_builder.rb +19 -2
- data/lib/karafka/pro/processing/jobs_queue.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/consumer.rb +20 -3
- data/lib/karafka/pro/processing/offset_metadata/fetcher.rb +19 -2
- data/lib/karafka/pro/processing/offset_metadata/listener.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/base.rb +25 -8
- data/lib/karafka/pro/processing/parallel_segments/filters/default.rb +19 -2
- data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb +19 -2
- data/lib/karafka/pro/processing/partitioner.rb +22 -5
- data/lib/karafka/pro/processing/periodic_job/consumer.rb +25 -7
- data/lib/karafka/pro/processing/piping/consumer.rb +26 -9
- data/lib/karafka/pro/processing/schedulers/base.rb +24 -7
- data/lib/karafka/pro/processing/schedulers/default.rb +24 -7
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb +25 -5
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb +21 -4
- data/lib/karafka/pro/processing/strategies/aj/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/base.rb +19 -2
- data/lib/karafka/pro/processing/strategies/default.rb +41 -24
- data/lib/karafka/pro/processing/strategies/dlq/default.rb +26 -9
- data/lib/karafka/pro/processing/strategies/dlq/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/dlq/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/ftr/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/ftr/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/default.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/ftr.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb +25 -5
- data/lib/karafka/pro/processing/strategies/lrj/ftr_mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/mom.rb +21 -4
- data/lib/karafka/pro/processing/strategies/lrj/mom_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/lrj/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/ftr_vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/mom/vp.rb +19 -2
- data/lib/karafka/pro/processing/strategies/vp/default.rb +19 -2
- data/lib/karafka/pro/processing/strategies.rb +19 -2
- data/lib/karafka/pro/processing/strategy_selector.rb +19 -2
- data/lib/karafka/pro/processing/subscription_groups_coordinator.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/balanced.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/base.rb +19 -2
- data/lib/karafka/pro/processing/virtual_partitions/distributors/consistent.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/consumer.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/config.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/contracts/task.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/deserializer.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/dispatcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/errors.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/executor.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/listener.rb +20 -3
- data/lib/karafka/pro/recurring_tasks/matcher.rb +21 -4
- data/lib/karafka/pro/recurring_tasks/schedule.rb +19 -2
- data/lib/karafka/pro/recurring_tasks/serializer.rb +24 -7
- data/lib/karafka/pro/recurring_tasks/setup/config.rb +22 -5
- data/lib/karafka/pro/recurring_tasks/task.rb +23 -6
- data/lib/karafka/pro/recurring_tasks.rb +23 -6
- data/lib/karafka/pro/routing/features/active_job/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/active_job.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/config.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/adaptive_iterator/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/adaptive_iterator.rb +19 -2
- data/lib/karafka/pro/routing/features/base.rb +19 -2
- data/lib/karafka/pro/routing/features/dead_letter_queue/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/dead_letter_queue/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/dead_letter_queue.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/config.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/delaying/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/delaying.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/direct_assignments/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/direct_assignments.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/config.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/expiring/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/expiring.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/config.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/filtering/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/filtering.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/config.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/inline_insights/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/inline_insights.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/long_running_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/long_running_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/long_running_job.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/multiplexing/patches/contracts/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_group.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing/subscription_groups_builder.rb +19 -2
- data/lib/karafka/pro/routing/features/multiplexing.rb +24 -7
- data/lib/karafka/pro/routing/features/non_blocking_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/non_blocking_job.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/config.rb +19 -2
- data/lib/karafka/pro/routing/features/offset_metadata/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/offset_metadata/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/offset_metadata.rb +20 -3
- data/lib/karafka/pro/routing/features/parallel_segments/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/config.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb +24 -7
- data/lib/karafka/pro/routing/features/parallel_segments/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/parallel_segments/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/parallel_segments.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/builder.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/config.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/consumer_group.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb +22 -5
- data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/patterns/detector.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/pattern.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/patterns/topics.rb +19 -2
- data/lib/karafka/pro/routing/features/patterns.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/config.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/pausing/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/pausing.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/config.rb +19 -2
- data/lib/karafka/pro/routing/features/periodic_job/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/periodic_job/topic.rb +20 -3
- data/lib/karafka/pro/routing/features/periodic_job.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/builder.rb +26 -9
- data/lib/karafka/pro/routing/features/recurring_tasks/config.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/recurring_tasks/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/recurring_tasks.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/builder.rb +32 -15
- data/lib/karafka/pro/routing/features/scheduled_messages/config.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/scheduled_messages/proxy.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/scheduled_messages.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/config.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm/contracts/routing.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/swarm/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/swarm.rb +20 -3
- data/lib/karafka/pro/routing/features/throttling/config.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/throttling/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/throttling.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions/config.rb +26 -9
- data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb +21 -4
- data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb +19 -2
- data/lib/karafka/pro/routing/features/virtual_partitions.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/consumer.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/contracts/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/contracts/message.rb +29 -12
- data/lib/karafka/pro/scheduled_messages/daily_buffer.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/day.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb +23 -6
- data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/dispatcher.rb +24 -7
- data/lib/karafka/pro/scheduled_messages/errors.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/max_epoch.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/proxy.rb +27 -10
- data/lib/karafka/pro/scheduled_messages/schema_validator.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/serializer.rb +19 -2
- data/lib/karafka/pro/scheduled_messages/setup/config.rb +21 -4
- data/lib/karafka/pro/scheduled_messages/state.rb +20 -3
- data/lib/karafka/pro/scheduled_messages/tracker.rb +21 -4
- data/lib/karafka/pro/scheduled_messages.rb +21 -4
- data/lib/karafka/pro/swarm/liveness_listener.rb +21 -4
- data/lib/karafka/process.rb +1 -1
- data/lib/karafka/processing/coordinator.rb +1 -1
- data/lib/karafka/processing/inline_insights/consumer.rb +4 -4
- data/lib/karafka/processing/inline_insights/tracker.rb +6 -6
- data/lib/karafka/processing/jobs/base.rb +6 -4
- data/lib/karafka/processing/schedulers/default.rb +4 -4
- data/lib/karafka/processing/strategies/base.rb +6 -6
- data/lib/karafka/processing/strategies/default.rb +13 -13
- data/lib/karafka/processing/strategies/dlq.rb +1 -1
- data/lib/karafka/processing/worker.rb +5 -5
- data/lib/karafka/railtie.rb +11 -11
- data/lib/karafka/routing/builder.rb +3 -3
- data/lib/karafka/routing/contracts/consumer_group.rb +6 -6
- data/lib/karafka/routing/contracts/routing.rb +2 -2
- data/lib/karafka/routing/contracts/topic.rb +4 -4
- data/lib/karafka/routing/features/active_job/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/base/expander.rb +4 -4
- data/lib/karafka/routing/features/base.rb +8 -8
- data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/declaratives/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/deserializers/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/eofed/contracts/topic.rb +3 -3
- data/lib/karafka/routing/features/inline_insights/contracts/topic.rb +2 -2
- data/lib/karafka/routing/features/inline_insights.rb +7 -7
- data/lib/karafka/routing/features/manual_offset_management/contracts/topic.rb +2 -2
- data/lib/karafka/routing/subscription_group.rb +9 -9
- data/lib/karafka/runner.rb +3 -3
- data/lib/karafka/server.rb +5 -5
- data/lib/karafka/setup/attributes_map.rb +7 -7
- data/lib/karafka/setup/config.rb +11 -11
- data/lib/karafka/setup/contracts/config.rb +2 -2
- data/lib/karafka/setup/defaults_injector.rb +11 -11
- data/lib/karafka/swarm/manager.rb +6 -6
- data/lib/karafka/swarm/node.rb +7 -7
- data/lib/karafka/swarm/supervisor.rb +6 -6
- data/lib/karafka/swarm.rb +1 -1
- data/lib/karafka/time_trackers/pause.rb +1 -1
- data/lib/karafka/version.rb +1 -1
- data/lib/karafka.rb +36 -36
- metadata +7 -4
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -25,8 +42,8 @@ module Karafka
|
|
|
25
42
|
@mutex = Mutex.new
|
|
26
43
|
@changes = Hash.new do |h, k|
|
|
27
44
|
h[k] = {
|
|
28
|
-
state:
|
|
29
|
-
join_state:
|
|
45
|
+
state: "",
|
|
46
|
+
join_state: "",
|
|
30
47
|
state_age: 0,
|
|
31
48
|
changed_at: monotonic_now
|
|
32
49
|
}
|
|
@@ -69,9 +86,9 @@ module Karafka
|
|
|
69
86
|
times = []
|
|
70
87
|
# stateage is in microseconds
|
|
71
88
|
# We monitor broker changes to make sure we do not introduce extra friction
|
|
72
|
-
times << (statistics[
|
|
73
|
-
times << statistics[
|
|
74
|
-
times << statistics[
|
|
89
|
+
times << (statistics["brokers"].values.map { |stats| stats["stateage"] }.min / 1_000)
|
|
90
|
+
times << statistics["cgrp"]["rebalance_age"]
|
|
91
|
+
times << statistics["cgrp"]["stateage"]
|
|
75
92
|
|
|
76
93
|
# Keep the previous change age for changes that were triggered by us
|
|
77
94
|
previous_changed_at = @changes[subscription_group_id][:changed_at]
|
|
@@ -79,8 +96,8 @@ module Karafka
|
|
|
79
96
|
@changes[subscription_group_id].merge!(
|
|
80
97
|
state_age: times.min,
|
|
81
98
|
changed_at: previous_changed_at,
|
|
82
|
-
join_state: statistics[
|
|
83
|
-
state: statistics[
|
|
99
|
+
join_state: statistics["cgrp"]["join_state"],
|
|
100
|
+
state: statistics["cgrp"]["state"]
|
|
84
101
|
)
|
|
85
102
|
end
|
|
86
103
|
|
|
@@ -181,11 +198,11 @@ module Karafka
|
|
|
181
198
|
# partitions assigned in sgs that can be scaled. If that is the case, we scale up.
|
|
182
199
|
def scale_up
|
|
183
200
|
multi_part_sgs_families = Karafka::App
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
201
|
+
.assignments
|
|
202
|
+
.select { |_, partitions| partitions.size > 1 }
|
|
203
|
+
.keys
|
|
204
|
+
.map { |sg| sg.subscription_group.name }
|
|
205
|
+
.uniq
|
|
189
206
|
|
|
190
207
|
# Select connections for scaling up
|
|
191
208
|
in_sg_families do |first_subscription_group, sg_listeners|
|
|
@@ -239,8 +256,8 @@ module Karafka
|
|
|
239
256
|
|
|
240
257
|
state[:state_age] >= @scale_delay &&
|
|
241
258
|
(monotonic_now - state[:changed_at]) >= @scale_delay &&
|
|
242
|
-
state[:state] ==
|
|
243
|
-
state[:join_state] ==
|
|
259
|
+
state[:state] == "up" &&
|
|
260
|
+
state[:join_state] == "steady"
|
|
244
261
|
end
|
|
245
262
|
end
|
|
246
263
|
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -12,8 +29,8 @@ module Karafka
|
|
|
12
29
|
class Config < Karafka::Contracts::Base
|
|
13
30
|
configure do |config|
|
|
14
31
|
config.error_messages = YAML.safe_load_file(
|
|
15
|
-
File.join(Karafka.gem_root,
|
|
16
|
-
).fetch(
|
|
32
|
+
File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
|
|
33
|
+
).fetch("en").fetch("validations").fetch("setup").fetch("config")
|
|
17
34
|
end
|
|
18
35
|
|
|
19
36
|
nested(:encryption) do
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -24,12 +41,12 @@ module Karafka
|
|
|
24
41
|
payload = message[:payload]
|
|
25
42
|
|
|
26
43
|
message[:headers] ||= {}
|
|
27
|
-
message[:headers][
|
|
44
|
+
message[:headers]["encryption"] = version
|
|
28
45
|
message[:payload] = cipher.encrypt(payload)
|
|
29
46
|
|
|
30
47
|
return message unless fingerprinter
|
|
31
48
|
|
|
32
|
-
message[:headers][
|
|
49
|
+
message[:headers]["encryption_fingerprint"] = fingerprinter.hexdigest(payload)
|
|
33
50
|
|
|
34
51
|
message
|
|
35
52
|
end
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -22,8 +39,8 @@ module Karafka
|
|
|
22
39
|
# @return [Object] deserialized payload
|
|
23
40
|
def call(message)
|
|
24
41
|
headers = message.headers
|
|
25
|
-
encryption = headers[
|
|
26
|
-
fingerprint = headers[
|
|
42
|
+
encryption = headers["encryption"]
|
|
43
|
+
fingerprint = headers["encryption_fingerprint"]
|
|
27
44
|
|
|
28
45
|
return super unless active && encryption
|
|
29
46
|
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -18,12 +35,12 @@ module Karafka
|
|
|
18
35
|
# Supporting versions allows us to be able to rotate private and public keys in case
|
|
19
36
|
# we would need this. We can increase the version, rotate and Karafka when decrypting
|
|
20
37
|
# will figure out proper private key based on the version
|
|
21
|
-
setting(:version, default:
|
|
38
|
+
setting(:version, default: "1")
|
|
22
39
|
|
|
23
40
|
# We always support one public key for producing messages
|
|
24
41
|
# Public key needs to be always present even if we do not plan to produce messages from
|
|
25
42
|
# a Karafka process. This is because of the web-ui and potentially other cases like this
|
|
26
|
-
setting(:public_key, default:
|
|
43
|
+
setting(:public_key, default: "")
|
|
27
44
|
|
|
28
45
|
# Private keys in pem format, where the key is the version and value is the key.
|
|
29
46
|
# This allows us to support key rotation
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -160,8 +177,8 @@ module Karafka
|
|
|
160
177
|
|
|
161
178
|
next unless SUPPORTED_NAMED_POSITIONS.include?(named_offset)
|
|
162
179
|
|
|
163
|
-
@mapped_topics[name][partition] = -1 if named_offset ==
|
|
164
|
-
@mapped_topics[name][partition] = -2 if named_offset ==
|
|
180
|
+
@mapped_topics[name][partition] = -1 if named_offset == "latest"
|
|
181
|
+
@mapped_topics[name][partition] = -2 if named_offset == "earliest"
|
|
165
182
|
end
|
|
166
183
|
end
|
|
167
184
|
end
|
data/lib/karafka/pro/iterator.rb
CHANGED
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -37,7 +54,7 @@ module Karafka
|
|
|
37
54
|
# so we don't stop polling data even when reaching the end (end on a given moment)
|
|
38
55
|
def initialize(
|
|
39
56
|
topics,
|
|
40
|
-
settings: {
|
|
57
|
+
settings: { "auto.offset.reset": "beginning" },
|
|
41
58
|
yield_nil: false,
|
|
42
59
|
max_wait_time: 200
|
|
43
60
|
)
|
data/lib/karafka/pro/loader.rb
CHANGED
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -20,6 +37,7 @@ module Karafka
|
|
|
20
37
|
encryption/setup/config
|
|
21
38
|
encryption/contracts/config
|
|
22
39
|
encryption/messages/parser
|
|
40
|
+
cli/topics
|
|
23
41
|
].freeze
|
|
24
42
|
|
|
25
43
|
# Zeitwerk pro loader
|
|
@@ -33,7 +51,7 @@ module Karafka
|
|
|
33
51
|
def require_all
|
|
34
52
|
FORCE_LOADED.each { |file| require_relative(file) }
|
|
35
53
|
|
|
36
|
-
PRO_LOADER.push_dir(Karafka.core_root.join(
|
|
54
|
+
PRO_LOADER.push_dir(Karafka.core_root.join("pro"), namespace: Karafka::Pro)
|
|
37
55
|
PRO_LOADER.setup
|
|
38
56
|
PRO_LOADER.eager_load
|
|
39
57
|
end
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -19,7 +36,7 @@ module Karafka
|
|
|
19
36
|
tracker = Tracker.new(
|
|
20
37
|
adi_config.safety_margin,
|
|
21
38
|
coordinator.last_polled_at,
|
|
22
|
-
topic.subscription_group.kafka.fetch(:
|
|
39
|
+
topic.subscription_group.kafka.fetch(:"max.poll.interval.ms")
|
|
23
40
|
)
|
|
24
41
|
|
|
25
42
|
messages.each(*args) do |message|
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|
|
@@ -1,7 +1,24 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
#
|
|
3
|
+
# Karafka Pro - Source Available Commercial Software
|
|
4
|
+
# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# This software is NOT open source. It is source-available commercial software
|
|
7
|
+
# requiring a paid license for use. It is NOT covered by LGPL.
|
|
8
|
+
#
|
|
9
|
+
# PROHIBITED:
|
|
10
|
+
# - Use without a valid commercial license
|
|
11
|
+
# - Redistribution, modification, or derivative works without authorization
|
|
12
|
+
# - Use as training data for AI/ML models or inclusion in datasets
|
|
13
|
+
# - Scraping, crawling, or automated collection for any purpose
|
|
14
|
+
#
|
|
15
|
+
# PERMITTED:
|
|
16
|
+
# - Reading, referencing, and linking for personal or commercial use
|
|
17
|
+
# - Runtime retrieval by AI assistants, coding agents, and RAG systems
|
|
18
|
+
# for the purpose of providing contextual help to Karafka users
|
|
19
|
+
#
|
|
20
|
+
# License: https://karafka.io/docs/Pro-License-Comm/
|
|
21
|
+
# Contact: contact@karafka.io
|
|
5
22
|
|
|
6
23
|
module Karafka
|
|
7
24
|
module Pro
|