karafka 2.4.18 → 2.5.0.beta2

data/bin/rspecs

@@ -3,8 +3,20 @@
 set -e
 
 # Run only regular non-forking specs first
-SPECS_TYPE=regular bundle exec rspec --tag ~type:pro --tag ~mode:fork
+SPECS_TYPE=regular bundle exec rspec \
+  --tag ~type:pro \
+  --tag ~mode:fork \
+  --exclude-pattern "**/pro/**/*_spec.rb" \
+  spec/lib/
+
 # Run forking specs, they need to run in isolation not to crash because of librdkafka
-SPECS_TYPE=regular bundle exec rspec --tag mode:fork
+SPECS_TYPE=regular bundle exec rspec \
+  --tag mode:fork \
+  --exclude-pattern "**/pro/**/*_spec.rb" \
+  spec/lib/
+
 # Run pro specs at the end
-SPECS_TYPE=pro bundle exec rspec --tag type:pro --tag ~mode:fork
+SPECS_TYPE=pro bundle exec rspec \
+  --tag type:pro \
+  --tag ~mode:fork \
+  spec/lib/

data/bin/verify_kafka_warnings

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Checks Kafka logs for unsupported warning patterns
+# Only specified warnings are allowed, all others should trigger failure
+
+allowed_patterns=(
+   "Performing controller activation"
+   "registered with feature metadata.version"
+   "Replayed TopicRecord for"
+   "Replayed PartitionRecord for"
+   "Previous leader None and previous leader epoch"
+   "Creating new"
+)
+
+# Get all warnings
+warnings=$(docker logs --since=0 kafka | grep WARN)
+exit_code=0
+
+while IFS= read -r line; do
+   allowed=0
+   for pattern in "${allowed_patterns[@]}"; do
+       if echo "$line" | grep -q "$pattern"; then
+           allowed=1
+           break
+       fi
+   done
+
+   if [ $allowed -eq 0 ]; then
+       echo "Unexpected warning found:"
+       echo "$line"
+       exit_code=1
+   fi
+done <<< "$warnings"
+
+exit $exit_code

data/bin/verify_topics_naming

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+# This script verifies that we do not create (except few needed exceptions) test topics that do
+# not start with the "it-" prefix which is our standard.
+#
+# This ensures that we can clearly identify all test topics for removal in case of doing dev work
+# on a long-lived Kafka cluster without option to fully reset it.
+#
+# It also ensures we have one convention that we can follow.
+
+require_relative '../spec/integrations_helper.rb'
+
+setup_karafka
+
+# Please note that "__" starting topics are not here by default. It is expected.
+invalid = Karafka::Admin
+          .cluster_info
+          .topics
+          .map { |topic| topic[:topic_name] }
+          .select { |topic| !topic.start_with?('it-') }
+          .select { |topic| topic.length <= 6 }
+
+invalid.each do |invalid_name|
+  puts "#{invalid_name} does not start with the \"it-\" prefix"
+end
+
+exit invalid.empty? ? 0 : 1

data/config/locales/errors.yml

@@ -24,6 +24,7 @@ en:
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
+      worker_thread_priority_format: must be between -3 and 3
 
       oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
 
@@ -36,6 +37,7 @@ en:
       internal.processing.expansions_selector_format: cannot be nil
       internal.processing.executor_class_format: cannot be nil
       internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
+      internal.processing.errors_tracker_class_format: 'needs to be nil or a class'
 
       internal.active_job.dispatcher_format: cannot be nil
       internal.active_job.job_options_contract_format: cannot be nil
@@ -67,6 +69,7 @@ en:
       internal.connection.proxy.metadata.timeout_format: needs to be an integer bigger than 0
       internal.connection.proxy.metadata.max_attempts_format: needs to be an integer bigger than 0
       internal.connection.proxy.metadata.wait_time_format: needs to be an integer bigger than 0
+      internal.connection.listener_thread_priority_format: must be between -3 and 3
 
       internal.swarm.manager_format: cannot be nil
       internal.swarm.orphaned_exit_code_format: needs to be an integer bigger or equal to 0

data/config/locales/pro_errors.yml

@@ -5,6 +5,7 @@ en:
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
       virtual_partitions.offset_metadata_strategy_format: needs to be either :exact or :current
       virtual_partitions.reducer_format: "needs to respond to `#call`"
+      virtual_partitions.distribution_format: "needs to be either :consistent or :balanced"
 
       long_running_job.active_format: needs to be either true or false
 
@@ -58,6 +59,8 @@ en:
       subscription_group_details_multiplexing_boot_mismatch: 'boot needs to be between min and max'
       subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
       subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
+      subscription_group_details_multiplexing_one_not_enough: 'min and max cannot equal 1'
+      subscription_group_details.multiplexing_scale_delay_format: 'needs to be an integer equal or more than 1000'
 
       swarm.active_format: needs to be true
       swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
@@ -88,8 +91,16 @@ en:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present
       patterns_regexps_not_unique: 'must be unique within consumer group'
+
       direct_assignments_homogenous: 'single consumer group cannot mix regular and direct assignments'
 
+      parallel_segments.partitioner_format: needs to be defined and needs to respond to `#call`
+      parallel_segments.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      parallel_segments.count_format: needs to be equal or more than 1
+      parallel_segments.active_format: needs to be boolean
+      parallel_segments.reducer_format: "needs to respond to `#call`"
+      parallel_segments.merge_key_format: "needs to be a non-empty string"
+
     pattern:
       regexp_format: must be a regular expression
       name_format: 'needs to be a string with a Kafka accepted format'
@@ -113,8 +124,8 @@ en:
 
       recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
       recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.schedules_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.logs_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.schedules.name_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.logs.name_format: 'needs to be a string with a Kafka accepted format'
       recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
       recurring_tasks.deserializer_format: 'needs to be configured'
       recurring_tasks.logging_format: needs to be a boolean

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.8.0
+    image: confluentinc/cp-kafka:7.9.1
 
     ports:
       - 9092:9092