karafka 2.4.18 → 2.5.0

data/bin/integrations

@@ -45,23 +45,29 @@ class Scenario
     'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
     'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
     'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze,
+    'swarm/with_blocking_at_exit_spec.rb' => [2].freeze,
     'instrumentation/post_errors_instrumentation_error_spec.rb' => [1].freeze,
     'cli/declaratives/delete/existing_with_exit_code_spec.rb' => [2].freeze,
     'cli/declaratives/create/new_with_exit_code_spec.rb' => [2].freeze,
-    'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze
+    'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze,
+    'cli/declaratives/align/incorrectly_spec.rb' => [1].freeze
   }.freeze
 
   private_constant :MAX_RUN_TIME, :EXIT_CODES
 
+  attr_reader :index
+
   # Creates scenario instance and runs in the background process
   #
   # @param path [String] path to the scenarios file
-  def initialize(path)
+  def initialize(path, index)
     @path = path
     # First 1024 characters from stdout
     @stdout_head = ''
     # Last 1024 characters from stdout
     @stdout_tail = ''
+    # Assigns the index for parallel execution in the CI if requested
+    @index = index
   end
 
   # Starts running given scenario in a separate process
@@ -252,16 +258,24 @@ specs.delete_if do |spec|
   false
 end
 
-raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
-
 # Randomize order
-seed = (ENV['SEED'] || rand(0..10_000)).to_i
+seed = (ENV['SPECS_SEED'] || rand(0..10_000)).to_i
+group = (ENV['SPECS_GROUP'] || -1).to_i
+groups = (ENV['SPECS_GROUPS'] || 2).to_i
 
 puts "Random seed: #{seed}"
+puts "Group: #{group}"
+puts "Groups: #{groups}"
 
 scenarios = specs
             .shuffle(random: Random.new(seed))
-            .map { |integration_test| Scenario.new(integration_test) }
+            .map
+            .with_index { |integration, index| Scenario.new(integration, index % groups) }
+            .delete_if { |scenario| scenario.index != group && group != -1 }
+
+raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if scenarios.empty?
+
+puts "Running #{scenarios.size} scenarios"
 
 regulars = scenarios.reject(&:linear?)
 linears = scenarios - regulars

data/bin/rspecs

@@ -3,8 +3,20 @@
 set -e
 
 # Run only regular non-forking specs first
-SPECS_TYPE=regular bundle exec rspec --tag ~type:pro --tag ~mode:fork
+SPECS_TYPE=regular bundle exec rspec \
+  --tag ~type:pro \
+  --tag ~mode:fork \
+  --exclude-pattern "**/pro/**/*_spec.rb" \
+  spec/lib/
+
 # Run forking specs, they need to run in isolation not to crash because of librdkafka
-SPECS_TYPE=regular bundle exec rspec --tag mode:fork
+SPECS_TYPE=regular bundle exec rspec \
+  --tag mode:fork \
+  --exclude-pattern "**/pro/**/*_spec.rb" \
+  spec/lib/
+
 # Run pro specs at the end
-SPECS_TYPE=pro bundle exec rspec --tag type:pro --tag ~mode:fork
+SPECS_TYPE=pro bundle exec rspec \
+  --tag type:pro \
+  --tag ~mode:fork \
+  spec/lib/

data/bin/verify_kafka_warnings

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Checks Kafka logs for unsupported warning patterns
+# Only specified warnings are allowed, all others should trigger failure
+
+allowed_patterns=(
+   "Performing controller activation"
+   "registered with feature metadata.version"
+   "Replayed TopicRecord for"
+   "Replayed PartitionRecord for"
+   "Previous leader None and previous leader epoch"
+   "Creating new"
+)
+
+# Get all warnings
+warnings=$(docker logs --since=0 kafka | grep WARN)
+exit_code=0
+
+while IFS= read -r line; do
+   allowed=0
+   for pattern in "${allowed_patterns[@]}"; do
+       if echo "$line" | grep -q "$pattern"; then
+           allowed=1
+           break
+       fi
+   done
+
+   if [ $allowed -eq 0 ]; then
+       echo "Unexpected warning found:"
+       echo "$line"
+       exit_code=1
+   fi
+done <<< "$warnings"
+
+exit $exit_code

data/bin/verify_topics_naming

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+# This script verifies that we do not create (except few needed exceptions) test topics that do
+# not start with the "it-" prefix which is our standard.
+#
+# This ensures that we can clearly identify all test topics for removal in case of doing dev work
+# on a long-lived Kafka cluster without option to fully reset it.
+#
+# It also ensures we have one convention that we can follow.
+
+require_relative '../spec/integrations_helper.rb'
+
+setup_karafka
+
+# Please note that "__" starting topics are not here by default. It is expected.
+invalid = Karafka::Admin
+          .cluster_info
+          .topics
+          .map { |topic| topic[:topic_name] }
+          .select { |topic| !topic.start_with?('it-') }
+          .select { |topic| topic.length <= 6 }
+
+invalid.each do |invalid_name|
+  puts "#{invalid_name} does not start with the \"it-\" prefix"
+end
+
+exit invalid.empty? ? 0 : 1

data/config/locales/errors.yml

@@ -24,6 +24,7 @@ en:
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
+      worker_thread_priority_format: must be between -3 and 3
 
       oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
 
@@ -36,6 +37,7 @@ en:
       internal.processing.expansions_selector_format: cannot be nil
       internal.processing.executor_class_format: cannot be nil
       internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
+      internal.processing.errors_tracker_class_format: 'needs to be nil or a class'
 
       internal.active_job.dispatcher_format: cannot be nil
       internal.active_job.job_options_contract_format: cannot be nil
@@ -67,6 +69,7 @@ en:
       internal.connection.proxy.metadata.timeout_format: needs to be an integer bigger than 0
       internal.connection.proxy.metadata.max_attempts_format: needs to be an integer bigger than 0
       internal.connection.proxy.metadata.wait_time_format: needs to be an integer bigger than 0
+      internal.connection.listener_thread_priority_format: must be between -3 and 3
 
       internal.swarm.manager_format: cannot be nil
       internal.swarm.orphaned_exit_code_format: needs to be an integer bigger or equal to 0
@@ -81,7 +84,8 @@ en:
       admin.kafka_format: needs to be a hash
       admin.group_id_format: 'needs to be a string with a Kafka accepted format'
       admin.max_wait_time_format: 'needs to be an integer bigger than 0'
-      admin.max_attempts_format: 'needs to be an integer bigger than 0'
+      admin.retry_backoff_format: 'needs to be an integer bigger than 100'
+      admin.max_retries_duration_format: 'needs to be an integer bigger than 1000'
 
       swarm.nodes_format: 'needs to be an integer bigger than 0'
       swarm.node_format: needs to be false or node instance

data/config/locales/pro_errors.yml

@@ -5,6 +5,7 @@ en:
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
       virtual_partitions.offset_metadata_strategy_format: needs to be either :exact or :current
       virtual_partitions.reducer_format: "needs to respond to `#call`"
+      virtual_partitions.distribution_format: "needs to be either :consistent or :balanced"
 
       long_running_job.active_format: needs to be either true or false
 
@@ -58,6 +59,8 @@ en:
       subscription_group_details_multiplexing_boot_mismatch: 'boot needs to be between min and max'
       subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
       subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
+      subscription_group_details_multiplexing_one_not_enough: 'min and max cannot equal 1'
+      subscription_group_details.multiplexing_scale_delay_format: 'needs to be an integer equal or more than 1000'
 
       swarm.active_format: needs to be true
       swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
@@ -88,8 +91,16 @@ en:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present
       patterns_regexps_not_unique: 'must be unique within consumer group'
+
       direct_assignments_homogenous: 'single consumer group cannot mix regular and direct assignments'
 
+      parallel_segments.partitioner_format: needs to be defined and needs to respond to `#call`
+      parallel_segments.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      parallel_segments.count_format: needs to be equal or more than 1
+      parallel_segments.active_format: needs to be boolean
+      parallel_segments.reducer_format: "needs to respond to `#call`"
+      parallel_segments.merge_key_format: "needs to be a non-empty string"
+
     pattern:
       regexp_format: must be a regular expression
       name_format: 'needs to be a string with a Kafka accepted format'
@@ -113,8 +124,8 @@ en:
 
       recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
       recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.schedules_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.logs_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.schedules.name_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.logs.name_format: 'needs to be a string with a Kafka accepted format'
       recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
       recurring_tasks.deserializer_format: 'needs to be configured'
       recurring_tasks.logging_format: needs to be a boolean

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.8.0
+    image: confluentinc/cp-kafka:8.0.0
 
     ports:
       - 9092:9092