karafka 2.5.3 → 2.5.4.rc1

data/bin/integrations

@@ -1,341 +0,0 @@
-#!/usr/bin/env ruby
-
-# Runner to run integration specs in parallel
-
-# Part of integration specs run linear without bundler.
-# If we would run bundle exec when running this code, bundler would inject its own context
-# into them, messing things up heavily
-#
-# Types of specs:
-# - regular - can run in parallel, includes all the helpers
-# - pristine - cannot run in parallel, uses custom bundler but includes helpers
-# - poro - cannot run in parallel, uses custom bundler, does not include any helpers
-raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
-
-require 'open3'
-require 'fileutils'
-require 'pathname'
-require 'tmpdir'
-require 'etc'
-
-ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
-
-# How many child processes with integration specs do we want to run in parallel
-# When the value is high, there's a problem with thread allocation on Github CI, that is why
-# we limit it. Locally we can run a lot of those, as many of them have sleeps and do not use a lot
-# of CPU. Locally we also cannot go beyond certain limit due to how often and how many topics we
-# create in Kafka. With an overloaded system, we start getting timeouts.
-CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 3
-
-# How may bytes do we want to keep from the stdout in the buffer for when we need to print it
-MAX_BUFFER_OUTPUT = 307_200
-
-# Abstraction around a single test scenario execution process
-class Scenario
-  # How long a scenario can run before we kill it
-  # This is a fail-safe just in case something would hang
-  MAX_RUN_TIME = 5 * 60 # 5 minutes tops
-
-  # There are rare cases where Karafka may force shutdown for some of the integration cases
-  # This includes exactly those
-  EXIT_CODES = {
-    default: [0],
-    'consumption/worker_critical_error_behaviour_spec.rb' => [0, 2].freeze,
-    'shutdown/on_hanging_jobs_and_a_shutdown_spec.rb' => [2].freeze,
-    'shutdown/on_hanging_on_shutdown_job_and_a_shutdown_spec.rb' => [2].freeze,
-    'shutdown/on_hanging_listener_and_shutdown_spec.rb' => [2].freeze,
-    'swarm/forceful_shutdown_of_hanging_spec.rb' => [2].freeze,
-    'swarm/with_blocking_at_exit_spec.rb' => [2].freeze,
-    # Segfault in the below spec can be expected because we pretty much force terminate handing
-    # C stuff. This spec is still useful as it catches other things
-    'instrumentation/post_errors_instrumentation_error_spec.rb' => [1, 139].freeze,
-    'cli/declaratives/delete/existing_with_exit_code_spec.rb' => [2].freeze,
-    'cli/declaratives/create/new_with_exit_code_spec.rb' => [2].freeze,
-    'cli/declaratives/plan/when_changes_with_detailed_exit_code_spec.rb' => [2].freeze,
-    'cli/declaratives/align/incorrectly_spec.rb' => [1].freeze,
-    'setup/with_kip_848_protocol_incorrect_config_spec.rb' => [1].freeze
-  }.freeze
-
-  private_constant :MAX_RUN_TIME, :EXIT_CODES
-
-  attr_reader :index
-
-  # Creates scenario instance and runs in the background process
-  #
-  # @param path [String] path to the scenarios file
-  def initialize(path, index)
-    @path = path
-    # First 1024 characters from stdout
-    @stdout_head = ''
-    # Last 1024 characters from stdout
-    @stdout_tail = ''
-    # Assigns the index for parallel execution in the CI if requested
-    @index = index
-  end
-
-  # Starts running given scenario in a separate process
-  def start
-    @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(init_and_build_cmd)
-    @started_at = current_time
-  end
-
-  # @return [String] integration spec name
-  def name
-    @path.gsub("#{ROOT_PATH}/spec/integrations/", '')
-  end
-
-  # @return [Symbol] type of spec
-  def type
-    scenario_dir = File.dirname(@path)
-
-    return :poro if scenario_dir.include?('_poro')
-    return :pristine if scenario_dir.include?('_pristine')
-
-    :regular
-  end
-
-  # @return [Boolean] any spec that is not a regular one should not run in parallel with others
-  def linear?
-    type != :regular
-  end
-
-  # @return [Boolean] did this scenario finished or is it still running
-  def finished?
-    # If the thread is running too long, kill it
-    if current_time - @started_at > MAX_RUN_TIME
-      begin
-        Process.kill('TERM', pid)
-      # It may finish right after we want to kill it, that's why we ignore this
-      rescue Errno::ESRCH
-      end
-    end
-
-    # We read it so it won't grow as we use our default logger that prints to both test.log and
-    # to stdout. Otherwise after reaching the buffer size, it would hang
-    buffer = ''
-    @stdout.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
-    @stdout_head = buffer if @stdout_head.empty?
-    @stdout_tail << buffer
-    @stdout_tail = @stdout_tail[-MAX_BUFFER_OUTPUT..-1] || @stdout_tail
-
-    !@wait_thr.alive?
-  end
-
-  # @return [Boolean] did this scenario finish successfully or not
-  def success?
-    expected_exit_codes = EXIT_CODES[name] || EXIT_CODES[:default]
-
-    expected_exit_codes.include?(exit_code)
-  end
-
-  # @return [Integer] pid of the process of this scenario
-  def pid
-    @wait_thr.pid
-  end
-
-  # @return [Integer] exit code of the process running given scenario
-  def exit_code
-    # There may be no exit status if we killed the thread
-    @wait_thr.value&.exitstatus || 123
-  end
-
-  # @return [String] exit status of the process
-  def exit_status
-    @wait_thr.value.to_s
-  end
-
-  # Prints a status report when scenario is finished and stdout if it failed
-  def report
-    if success?
-      print "\e[#{32}m#{'.'}\e[0m"
-    else
-      buffer = ''
-
-      @stderr.read_nonblock(MAX_BUFFER_OUTPUT, buffer, exception: false)
-
-      puts
-      puts "\e[#{31}m#{'[FAILED]'}\e[0m #{name}"
-      puts "Time taken: #{current_time - @started_at} seconds"
-      puts "Exit code: #{exit_code}"
-      puts "Exit status: #{exit_status}"
-      puts @stdout_head
-      puts '...'
-      puts @stdout_tail
-      puts buffer
-      puts
-    end
-  end
-
-  # @return [Float] number of seconds that a given spec took to run
-  def time_taken
-    @finished_at - @started_at
-  end
-
-  # Close all the files that are open, so they do not pile up
-  def close
-    @finished_at = current_time
-    @stdin.close
-    @stdout.close
-    @stderr.close
-  end
-
-  private
-
-  # Sets up a proper environment for a given spec to run and returns the run command
-  # @return [String] run command
-  def init_and_build_cmd
-    case type
-    when :poro
-      scenario_dir = File.dirname(@path)
-      # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
-      temp_dir = Dir.mktmpdir
-      file_name = File.basename(@path)
-
-      FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
-
-      <<~CMD
-        cd #{temp_dir} &&
-        KARAFKA_GEM_DIR=#{ROOT_PATH} \
-        BUNDLE_AUTO_INSTALL=true \
-        PRISTINE_MODE=true \
-        bundle exec ruby #{file_name}
-      CMD
-    when :pristine
-      scenario_dir = File.dirname(@path)
-      # We copy the spec into a temp dir, not to pollute the spec location with logs, etc
-      temp_dir = Dir.mktmpdir
-      file_name = File.basename(@path)
-
-      FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
-
-      <<~CMD
-        cd #{temp_dir} &&
-        KARAFKA_GEM_DIR=#{ROOT_PATH} \
-        BUNDLE_AUTO_INSTALL=true \
-        PRISTINE_MODE=true \
-        bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{file_name}
-      CMD
-    else
-      <<~CMD
-        KARAFKA_GEM_DIR=#{ROOT_PATH} \
-        bundle exec ruby -r ./spec/integrations_helper.rb #{@path}
-      CMD
-    end
-  end
-
-  # @return [Float] current machine time
-  def current_time
-    Process.clock_gettime(Process::CLOCK_MONOTONIC)
-  end
-end
-
-# Load all the specs
-specs = Dir[ROOT_PATH.join('spec/integrations/**/*_spec.rb')]
-
-FILTER_TYPE = ARGV[0] == '--exclude' ? 'exclude' : 'include'
-
-# Remove the exclude flag
-ARGV.shift if FILTER_TYPE == '--exclude'
-
-# If filters is provided, apply
-# Allows to provide several filters one after another and applies all of them
-ARGV.each do |filter|
-  specs.delete_if do |name|
-    case FILTER_TYPE
-    when 'include'
-      !name.include?(filter)
-    when 'exclude'
-      name.include?(filter)
-    else
-      raise 'Invalid filter type'
-    end
-  end
-end
-
-# Remove Rails 7.2 specs from Ruby < 3.1 because it requires 3.1
-# Remove Rails 8.0 specs from Ruby < 3.2 because it requires 3.2
-specs.delete_if do |spec|
-  next true if RUBY_VERSION < '3.1' && spec.include?('rails72')
-  next true if RUBY_VERSION < '3.2' && spec.include?('rails8')
-
-  false
-end
-
-# Randomize order
-seed = (ENV['SPECS_SEED'] || rand(0..10_000)).to_i
-group = (ENV['SPECS_GROUP'] || -1).to_i
-groups = (ENV['SPECS_GROUPS'] || 2).to_i
-
-puts "Random seed: #{seed}"
-puts "Group: #{group}"
-puts "Groups: #{groups}"
-
-scenarios = specs
-            .shuffle(random: Random.new(seed))
-            .map
-            .with_index { |integration, index| Scenario.new(integration, index % groups) }
-            .delete_if { |scenario| scenario.index != group && group != -1 }
-
-raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if scenarios.empty?
-
-puts "Running #{scenarios.size} scenarios"
-
-regulars = scenarios.reject(&:linear?)
-linears = scenarios - regulars
-
-active_scenarios = []
-finished_scenarios = []
-
-while finished_scenarios.size < scenarios.size
-  # If we have space to run another scenario, we add it
-  if active_scenarios.size < CONCURRENCY
-    scenario = nil
-    # We can run only one linear at the same time due to concurrency issues within bundler
-    # Since they usually take longer than others, we try to run them as fast as possible when there
-    # is a slot
-    scenario = linears.pop unless active_scenarios.any?(&:linear?)
-    scenario ||= regulars.pop
-
-    if scenario
-      scenario.start
-      active_scenarios << scenario
-    end
-  end
-
-  active_scenarios.select(&:finished?).each do |exited|
-    scenario = active_scenarios.delete(exited)
-    scenario.report
-    scenario.close
-    finished_scenarios << scenario
-  end
-
-  sleep(0.1)
-end
-
-# Report longest scenarios
-puts
-puts "\nLongest scenarios:\n\n"
-
-finished_scenarios.sort_by(&:time_taken).reverse.first(10).each do |long_scenario|
-  puts "[#{'%6.2f' % long_scenario.time_taken}] #{long_scenario.name}"
-end
-
-failed_scenarios = finished_scenarios.reject(&:success?)
-
-if failed_scenarios.empty?
-  puts
-else
-  # Report once more on the failed jobs
-  # This will only list scenarios that failed without printing their stdout here.
-  puts
-  puts "\nFailed scenarios:\n\n"
-
-  failed_scenarios.each do |scenario|
-    puts "\e[#{31}m#{'[FAILED]'}\e[0m #{scenario.name}"
-  end
-
-  puts
-
-  # Exit with 1 if not all scenarios were successful
-  exit 1
-end

data/bin/record_rss

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-# This script monitors and records the Resident Set Size (RSS) of a process given its PID.
-# The RSS is logged every second to the specified output file until the process terminates.
-#
-# Usage:
-# ./script_name.sh <PID> <OUTPUT_FILE>
-#
-# Arguments:
-# <PID>         - Process ID of the process you want to monitor.
-# <OUTPUT_FILE> - Name of the file where RSS values will be logged.
-#
-# The script first checks if the correct number of arguments are provided.
-# It then verifies if the given PID exists. If it does, it starts recording the RSS.
-# For every iteration, the script fetches the current RSS of the process using the 'ps' command,
-# then appends the RSS value along with a timestamp to the output file.
-# This recording is done every second.
-# The loop stops if the process with the given PID terminates.
-# An informative message is printed out when recording starts and when it stops.
-
-# Check if the correct number of arguments are passed
-if [ "$#" -ne 2 ]; then
-    echo "Usage: $0 <PID> <OUTPUT_FILE>"
-    exit 1
-fi
-
-PID=$1
-OUTPUT_FILE=$2
-
-# Check if the given PID exists
-if ! kill -0 $PID 2>/dev/null; then
-    echo "Error: PID $PID does not exist."
-    exit 1
-fi
-
-# Start recording the RSS
-echo "Recording RSS for PID $PID every second to $OUTPUT_FILE..."
-
-while kill -0 $PID 2>/dev/null; do
-    RSS=$(ps -o rss= -p $PID)
-    if [ -z "$RSS" ]; then
-        echo "Error: Failed to get RSS for PID $PID."
-        exit 1
-    fi
-    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
-    echo "$TIMESTAMP: $RSS KB" >> $OUTPUT_FILE
-    sleep 1
-done
-
-echo "Process $PID has terminated. Stopping recording."

data/bin/rspecs

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-# We remove old coverage because under heavy development and parallel spec execution, the
-# rspec results formatter tends to crash
-rm -rf ./coverage
-
-# Run only regular non-forking specs first
-SPECS_TYPE=regular bundle exec rspec \
-  --tag ~type:pro \
-  --tag ~mode:fork \
-  --exclude-pattern "**/pro/**/*_spec.rb" \
-  spec/lib/
-
-# Run forking specs, they need to run in isolation not to crash because of librdkafka
-SPECS_TYPE=regular bundle exec rspec \
-  --tag mode:fork \
-  --exclude-pattern "**/pro/**/*_spec.rb" \
-  spec/lib/
-
-# Run pro specs at the end
-SPECS_TYPE=pro bundle exec rspec \
-  --tag type:pro \
-  --tag ~mode:fork \
-  spec/lib/

data/bin/scenario

@@ -1,29 +0,0 @@
-#!/usr/bin/env ruby
-
-# Runner for non-parallel execution of a single scenario.
-# It prints all the info stdout, etc and basically replaces itself with the scenario execution.
-# It is useful when we work with a single spec and we need all the debug info
-
-raise 'This code needs to be executed WITHOUT bundle exec' if Kernel.const_defined?(:Bundler)
-
-require 'open3'
-require 'fileutils'
-require 'pathname'
-require 'tmpdir'
-require 'etc'
-
-ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../')))
-
-# Load all the specs
-specs = Dir[ROOT_PATH.join('spec/integrations/**/*.rb')]
-
-# If filters is provided, apply
-# Allows to provide several filters one after another and applies all of them
-ARGV.each do |filter|
-  specs.delete_if { |name| !name.include?(filter) }
-end
-
-raise ArgumentError, "No integration specs with filters: #{ARGV.join(', ')}" if specs.empty?
-raise ArgumentError, "Many specs found with filters: #{ARGV.join(', ')}" if specs.size != 1
-
-exec("bundle exec ruby -r #{ROOT_PATH}/spec/integrations_helper.rb #{specs[0]}")

data/bin/stress_many

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Runs integration specs in an endless loop
-# This allows us to ensure (after long enough time) that the integrations test suit is stable and
-# that there are no anomalies when running it for a long period of time
-
-set -e
-
-while :
-do
-  clear
-  bin/integrations $1
-done

data/bin/stress_one

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Runs a single integration spec in an endless loop
-# This allows us to ensure (after long enough time) that the integration spec is stable and
-# that there are no anomalies when running it for a long period of time
-
-set -e
-
-while :
-do
-  clear
-  bin/scenario $1
-done

data/bin/verify_kafka_warnings

@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Checks Kafka logs for unsupported warning patterns
-# Only specified warnings are allowed, all others should trigger failure
-
-allowed_patterns=(
-   "Performing controller activation"
-   "registered with feature metadata.version"
-   "Replayed TopicRecord for"
-   "Replayed PartitionRecord for"
-   "Previous leader None and previous leader epoch"
-   "Creating new"
-   "Unloaded transaction metadata"
-)
-
-# Get all warnings
-warnings=$(docker logs --since=0 kafka | grep "] WARN ")
-exit_code=0
-
-while IFS= read -r line; do
-   allowed=0
-   for pattern in "${allowed_patterns[@]}"; do
-       if echo "$line" | grep -q "$pattern"; then
-           allowed=1
-           break
-       fi
-   done
-
-   if [ $allowed -eq 0 ]; then
-       echo "Unexpected warning found:"
-       echo "$line"
-       exit_code=1
-   fi
-done <<< "$warnings"
-
-exit $exit_code

data/bin/verify_license_integrity

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-# This script verifies integrity of the Pro license
-# Run it before bundle install to ensure, that what you are fetching is what you expect
-# Run it after bundle install to ensure that the local artefact was not compromised
-
-#!/usr/bin/env bash
-
-set -e
-
-if [ "$MODE" != "after" ]; then
-  # Check the remote license prior to bundle installing
-  curl \
-    --retry 5 \
-    --retry-delay 1 \
-    --fail \
-    -u $KARAFKA_PRO_USERNAME:$KARAFKA_PRO_PASSWORD \
-    https://gems.karafka.io/gems/karafka-license-$KARAFKA_PRO_VERSION.gem \
-    -o ./karafka-license.gem
-else
-  # Check the local cached one after bundle install
-  cache_path=`ruby -e 'puts "#{Gem.dir}/cache/"'`
-  cp "$cache_path/karafka-license-$KARAFKA_PRO_VERSION.gem" ./karafka-license.gem
-fi
-
-detected=`sha256sum ./karafka-license.gem | awk '{ print $1 }'`
-
-rm ./karafka-license.gem
-
-echo -n "Karafka Pro license artifact checksum verification result: "
-
-if [ "$detected" = "$KARAFKA_PRO_LICENSE_CHECKSUM" ]; then
-  echo "Success"
-else
-  echo -e "\033[0;31mFailure!\033[0m"
-  exit 1
-fi

data/bin/verify_topics_naming

@@ -1,27 +0,0 @@
-#!/usr/bin/env ruby
-
-# This script verifies that we do not create (except few needed exceptions) test topics that do
-# not start with the "it-" prefix which is our standard.
-#
-# This ensures that we can clearly identify all test topics for removal in case of doing dev work
-# on a long-lived Kafka cluster without option to fully reset it.
-#
-# It also ensures we have one convention that we can follow.
-
-require_relative '../spec/integrations_helper.rb'
-
-setup_karafka
-
-# Please note that "__" starting topics are not here by default. It is expected.
-invalid = Karafka::Admin
-          .cluster_info
-          .topics
-          .map { |topic| topic[:topic_name] }
-          .select { |topic| !topic.start_with?('it-') }
-          .select { |topic| topic.length <= 6 }
-
-invalid.each do |invalid_name|
-  puts "#{invalid_name} does not start with the \"it-\" prefix"
-end
-
-exit invalid.empty? ? 0 : 1

data/bin/wait_for_kafka

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-
-# Waits for Kafka to be ready
-# Useful in CI where Kafka needs to be fully started before we run any tests
-
-require 'karafka'
-
-Karafka::App.setup do |config|
-  config.kafka[:'bootstrap.servers'] = '127.0.0.1:9092'
-end
-
-60.times do
-  begin
-    # Stop if we can connect to the cluster and get info
-    exit if Karafka::Admin.cluster_info
-  rescue Rdkafka::RdkafkaError
-    puts "Kafka not available, retrying..."
-    sleep(1)
-  end
-end
-
-puts 'Kafka not available!'
-
-exit 1

data/docker-compose.yml

@@ -1,25 +0,0 @@
-services:
-  kafka:
-    container_name: kafka
-    image: confluentinc/cp-kafka:8.1.0
-
-    ports:
-      - 9092:9092
-
-    environment:
-      CLUSTER_ID: kafka-docker-cluster-1
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_PROCESS_ROLES: broker,controller
-      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
-      KAFKA_BROKER_ID: 1
-      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
-      ALLOW_PLAINTEXT_LISTENER: 'yes'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
-      KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer