karafka-rdkafka 0.12.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fb0e8bec19a86f44453d32057d22c3958b4220e1d710564360b369905fd57d81
+  data.tar.gz: 645b3ab638f51780de30a21e133bd07b0e80fd33a1a59e604ec1dc064fc2b9c4
+SHA512:
+  metadata.gz: 52120cbb353e7bf7af881f442fb7a36324a70b097969af2f0e855f99e820c0be88f26ab584a285a85408eda526a0e564e06420e8c4d8224e40ae7d00f70546ab
+  data.tar.gz: ba886f70d88433d6dce7247df1c6d3598d20642fd1a4c17cb045f0249f22b7264c329462e0e4ae986fa053dd2a7c3598b0379fe4924822484d60c9b6baaecc35

checksums.yaml.gz.sig

@@ -0,0 +1,2 @@
+��t“qiK����2]T��poy[��C�􀚝M~Z���{��ү�����)��j�ںUM���^k��%Qv4��K�Ul3o2`T���
+�o7s�;7��=�y��gol��u

data/.gitignore

@@ -0,0 +1,8 @@
+Gemfile.lock
+ext/ports
+ext/tmp
+ext/librdkafka.*
+*.gem
+.yardoc
+doc
+coverage

data/.rspec

@@ -0,0 +1 @@
+--format documentation

data/.semaphore/semaphore.yml

@@ -0,0 +1,23 @@
+version: v1.0
+name: Rdkafka Ruby
+
+agent:
+  machine:
+    type: e1-standard-4
+    os_image: ubuntu1804
+
+blocks:
+  - name: Run specs
+    task:
+      jobs:
+        - name: bundle exec rspec
+          matrix:
+            - env_var: RUBY_VERSION
+              values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
+          commands:
+            - sem-version ruby $RUBY_VERSION
+            - checkout
+            - bundle install --path vendor/bundle
+            - cd ext && bundle exec rake && cd ..
+            - docker-compose up -d --no-recreate
+            - bundle exec rspec

data/.yardopts

@@ -0,0 +1,2 @@
+--no-private
+--markup=markdown

data/CHANGELOG.md

@@ -0,0 +1,104 @@
+# 0.12.0
+* Bumps librdkafka to 1.9.0
+
+# 0.11.0
+* Upgrade librdkafka to 1.8.2
+* Bump supported minimum Ruby version to 2.6
+* Better homebrew path detection
+
+# 0.10.0
+* Upgrade librdkafka to 1.5.0
+* Add error callback config
+
+# 0.9.0
+* Fixes for Ruby 3.0
+* Allow any callable object for callbacks (gremerritt)
+* Reduce memory allocations in Rdkafka::Producer#produce (jturkel)
+* Use queue as log callback to avoid unsafe calls from trap context (breunigs)
+* Allow passing in topic configuration on create_topic (dezka)
+* Add each_batch method to consumer (mgrosso)
+
+# 0.8.1
+* Fix topic_flag behaviour and add tests for Metadata (geoff2k)
+* Add topic admin interface (geoff2k)
+* Raise an exception if @native_kafka is nil (geoff2k)
+* Option to use zstd compression (jasonmartens)
+
+# 0.8.0
+* Upgrade librdkafka to 1.4.0
+* Integrate librdkafka metadata API and add partition_key (by Adithya-copart)
+* Ruby 2.7 compatibility fix (by Geoff Thé)A
+* Add error to delivery report (by Alex Stanovsky)
+* Don't override CPPFLAGS and LDFLAGS if already set on Mac (by Hiroshi Hatake)
+* Allow use of Rake 13.x and up (by Tomasz Pajor)
+
+# 0.7.0
+* Bump librdkafka to 1.2.0 (by rob-as)
+* Allow customizing the wait time for delivery report availability (by mensfeld)
+
+# 0.6.0
+* Bump librdkafka to 1.1.0 (by Chris Gaffney)
+* Implement seek (by breunigs)
+
+# 0.5.0
+* Bump librdkafka to 1.0.0 (by breunigs)
+* Add cluster and member information (by dmexe)
+* Support message headers for consumer & producer (by dmexe)
+* Add consumer rebalance listener (by dmexe)
+* Implement pause/resume partitions (by dmexe)
+
+# 0.4.2
+* Delivery callback for producer
+* Document list param of commit method
+* Use default Homebrew openssl location if present
+* Consumer lag handles empty topics
+* End iteration in consumer when it is closed
+* Add support for storing message offsets
+* Add missing runtime dependency to rake
+
+# 0.4.1
+* Bump librdkafka to 0.11.6
+
+# 0.4.0
+* Improvements in librdkafka archive download
+* Add global statistics callback
+* Use Time for timestamps, potentially breaking change if you
+  rely on the previous behavior where it returns an integer with
+  the number of milliseconds.
+* Bump librdkafka to 0.11.5
+* Implement TopicPartitionList in Ruby so we don't have to keep
+  track of native objects.
+* Support committing a topic partition list
+* Add consumer assignment method
+
+# 0.3.5
+* Fix crash when not waiting for delivery handles
+* Run specs on Ruby 2.5
+
+# 0.3.4
+* Bump librdkafka to 0.11.3
+
+# 0.3.3
+* Fix bug that prevent display of `RdkafkaError` message
+
+# 0.3.2
+* `add_topic` now supports using a partition count
+* Add way to make errors clearer with an extra message
+* Show topics in subscribe error message
+* Show partition and topic in query watermark offsets error message
+
+# 0.3.1
+* Bump librdkafka to 0.11.1
+* Officially support ranges in `add_topic` for topic partition list.
+* Add consumer lag calculator
+
+# 0.3.0
+* Move both add topic methods to one `add_topic` in `TopicPartitionList`
+* Add committed offsets to consumer
+* Add query watermark offset to consumer
+
+# 0.2.0
+* Some refactoring and add inline documentation
+
+# 0.1.x
+* Initial working version including producing and consuming

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Guardfile

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+logger level: :error
+
+guard :rspec, cmd: "bundle exec rspec --format #{ENV.fetch("FORMAT", "documentation")}" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+  watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+end

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Thijs Cadier
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,114 @@
+# Rdkafka
+
+[![Build Status](https://appsignal.semaphoreci.com/badges/rdkafka-ruby/branches/master.svg?style=shields)](https://appsignal.semaphoreci.com/projects/rdkafka-ruby)
+[![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
+[![Maintainability](https://api.codeclimate.com/v1/badges/ecb1765f81571cccdb0e/maintainability)](https://codeclimate.com/github/appsignal/rdkafka-ruby/maintainability)
+
+The `rdkafka` gem is a modern Kafka client library for Ruby based on
+[librdkafka](https://github.com/edenhill/librdkafka/).
+It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
+gem and targets Kafka 1.0+ and Ruby versions that are under security or
+active maintenance. We remove Ruby version from our CI builds if they 
+become EOL.
+
+`rdkafka` was written because we needed a reliable Ruby client for
+Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
+We run it in production on very high traffic systems.
+
+This gem only provides a high-level Kafka consumer. If you are running
+an older version of Kafka and/or need the legacy simple consumer we
+suggest using the [Hermann](https://github.com/reiseburo/hermann) gem.
+
+The most important pieces of a Kafka client are implemented. We're
+working towards feature completeness, you can track that here:
+https://github.com/appsignal/rdkafka-ruby/milestone/1
+
+## Installation
+
+This gem downloads and compiles librdkafka when it is installed. If you
+have any problems installing the gem please open an issue.
+
+## Usage
+
+See the [documentation](https://www.rubydoc.info/github/appsignal/rdkafka-ruby) for full details on how to use this gem. Two quick examples:
+
+### Consuming messages
+
+Subscribe to a topic and get messages. Kafka will automatically spread
+the available partitions over consumers with the same group id.
+
+```ruby
+config = {
+  :"bootstrap.servers" => "localhost:9092",
+  :"group.id" => "ruby-test"
+}
+consumer = Rdkafka::Config.new(config).consumer
+consumer.subscribe("ruby-test-topic")
+
+consumer.each do |message|
+  puts "Message received: #{message}"
+end
+```
+
+### Producing messages
+
+Produce a number of messages, put the delivery handles in an array and
+wait for them before exiting. This way the messages will be batched and
+sent to Kafka in an efficient way.
+
+```ruby
+config = {:"bootstrap.servers" => "localhost:9092"}
+producer = Rdkafka::Config.new(config).producer
+delivery_handles = []
+
+100.times do |i|
+  puts "Producing message #{i}"
+  delivery_handles << producer.produce(
+      topic:   "ruby-test-topic",
+      payload: "Payload #{i}",
+      key:     "Key #{i}"
+  )
+end
+
+delivery_handles.each(&:wait)
+```
+
+Note that creating a producer consumes some resources that will not be
+released until it `#close` is explicitly called, so be sure to call
+`Config#producer` only as necessary.
+
+## Development
+
+A Docker Compose file is included to run Kafka and Zookeeper. To run
+that:
+
+```
+docker-compose up
+```
+
+Run `bundle` and `cd ext && bundle exec rake && cd ..` to download and
+compile `librdkafka`.
+
+You can then run `bundle exec rspec` to run the tests. To see rdkafka
+debug output:
+
+```
+DEBUG_PRODUCER=true bundle exec rspec
+DEBUG_CONSUMER=true bundle exec rspec
+```
+
+After running the tests you can bring the cluster down to start with a
+clean slate:
+
+```
+docker-compose down
+```
+
+## Example
+
+To see everything working run these in separate tabs:
+
+```
+bundle exec rake consume_messages
+bundle exec rake produce_messages
+```

data/Rakefile

@@ -0,0 +1,96 @@
+# Rakefile
+
+require 'bundler/gem_tasks'
+require "./lib/rdkafka"
+
+desc 'Generate some message traffic'
+task :produce_messages do
+  config = {:"bootstrap.servers" => "localhost:9092"}
+  if ENV["DEBUG"]
+    config[:debug] = "broker,topic,msg"
+  end
+  producer = Rdkafka::Config.new(config).producer
+
+  delivery_handles = []
+  100.times do |i|
+    puts "Producing message #{i}"
+    delivery_handles << producer.produce(
+        topic:   "rake_test_topic",
+        payload: "Payload #{i} from Rake",
+        key:     "Key #{i} from Rake"
+    )
+  end
+  puts 'Waiting for delivery'
+  delivery_handles.each(&:wait)
+  puts 'Done'
+end
+
+desc 'Consume some messages'
+task :consume_messages do
+  config = {
+    :"bootstrap.servers" => "localhost:9092",
+    :"group.id" => "rake_test",
+    :"enable.partition.eof" => false,
+    :"auto.offset.reset" => "earliest",
+    :"statistics.interval.ms" => 10_000
+  }
+  if ENV["DEBUG"]
+    config[:debug] = "cgrp,topic,fetch"
+  end
+  Rdkafka::Config.statistics_callback = lambda do |stats|
+    puts stats
+  end
+  consumer = Rdkafka::Config.new(config).consumer
+  consumer = Rdkafka::Config.new(config).consumer
+  consumer.subscribe("rake_test_topic")
+  consumer.each do |message|
+    puts "Message received: #{message}"
+  end
+end
+
+desc 'Hammer down'
+task :load_test do
+  puts "Starting load test"
+
+  config = Rdkafka::Config.new(
+    :"bootstrap.servers" => "localhost:9092",
+    :"group.id" => "load-test",
+    :"enable.partition.eof" => false
+  )
+
+  # Create a producer in a thread
+  Thread.new do
+    producer = config.producer
+    loop do
+      handles = []
+      1000.times do |i|
+        handles.push(producer.produce(
+          topic:   "load_test_topic",
+          payload: "Payload #{i}",
+          key:     "Key #{i}"
+        ))
+      end
+      handles.each(&:wait)
+      puts "Produced 1000 messages"
+    end
+  end.abort_on_exception = true
+
+  # Create three consumers in threads
+  3.times do |i|
+    Thread.new do
+      count = 0
+      consumer = config.consumer
+      consumer.subscribe("load_test_topic")
+      consumer.each do |message|
+        count += 1
+        if count % 1000 == 0
+          puts "Received 1000 messages in thread #{i}"
+        end
+      end
+    end.abort_on_exception = true
+  end
+
+  loop do
+    sleep 1
+  end
+end

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
+
+require "bundler/setup"
+require "rdkafka"
+
+require "irb"
+IRB.start(__FILE__)

data/docker-compose.yml

@@ -0,0 +1,24 @@
+---
+
+version: '2'
+
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.2.6
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  kafka:
+    image: confluentinc/cp-kafka:5.2.5-10
+    depends_on:
+      - zookeeper
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -0,0 +1,18 @@
+# Ext
+
+This gem depends on the `librdkafka` C library. It is downloaded when
+this gem is installed.
+
+To update the `librdkafka` version follow the following steps:
+
+* Go to https://github.com/edenhill/librdkafka/releases to get the new
+  version number and asset checksum for `tar.gz`.
+* Change the version in `lib/rdkafka/version.rb`
+* Change the `sha256` in `lib/rdkafka/version.rb`
+* Run `bundle exec rake` in the `ext` directory to download and build
+  the new version
+* Run `docker-compose pull` in the main gem directory to ensure the docker
+  images used by the tests and run `docker-compose up`
+* Finally, run `bundle exec rspec` in the main gem directory to execute
+  the test suite to detect any regressions that may have been introduced
+  by the update

data/ext/Rakefile

@@ -0,0 +1,62 @@
+require File.expand_path('../../lib/rdkafka/version', __FILE__)
+require "mini_portile2"
+require "fileutils"
+require "open-uri"
+
+task :default => :clean do
+  # Download and compile librdkafka
+  recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+
+  # Use default homebrew openssl if we're on mac and the directory exists
+  # and each of flags is not empty
+  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+  end
+
+  recipe.files << {
+    :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+    :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+  }
+  recipe.configure_options = ["--host=#{recipe.host}"]
+  recipe.cook
+  # Move dynamic library we're interested in
+  if recipe.host.include?('darwin')
+    from_extension = '1.dylib'
+    to_extension   = 'dylib'
+  else
+    from_extension = 'so.1'
+    to_extension = 'so'
+  end
+  lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
+  FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
+  # Cleanup files created by miniportile we don't need in the gem
+  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
+  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
+end
+
+task :clean do
+  FileUtils.rm_f File.join(File.dirname(__FILE__), "librdkafka.dylib")
+  FileUtils.rm_f File.join(File.dirname(__FILE__), "librdkafka.so")
+  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
+  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
+end
+
+namespace :build do
+  desc "Build librdkafka at the given git sha or tag"
+  task :git, [:ref] do |task, args|
+    ref = args[:ref]
+    version = "git-#{ref}"
+
+    recipe = MiniPortile.new("librdkafka", version)
+    recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
+    recipe.configure_options = ["--host=#{recipe.host}","--enable-static", "--enable-zstd"]
+    recipe.cook
+
+    ext = recipe.host.include?("darwin") ? "dylib" : "so"
+    lib = File.expand_path("ports/#{recipe.host}/librdkafka/#{version}/lib/librdkafka.#{ext}", __dir__)
+
+    # Copy will copy the content, following any symlinks
+    FileUtils.cp(lib, __dir__)
+  end
+end

data/lib/rdkafka/abstract_handle.rb

@@ -0,0 +1,82 @@
+require "ffi"
+
+module Rdkafka
+  class AbstractHandle < FFI::Struct
+    # Subclasses must define their own layout, and the layout must start with:
+    #
+    # layout :pending, :bool,
+    #        :response, :int
+
+    REGISTRY = {}
+
+    CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
+
+    private_constant :CURRENT_TIME
+
+    def self.register(handle)
+      address = handle.to_ptr.address
+      REGISTRY[address] = handle
+    end
+
+    def self.remove(address)
+      REGISTRY.delete(address)
+    end
+
+    # Whether the handle is still pending.
+    #
+    # @return [Boolean]
+    def pending?
+      self[:pending]
+    end
+
+    # Wait for the operation to complete or raise an error if this takes longer than the timeout.
+    # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
+    # In this case it is possible to call wait again.
+    #
+    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
+    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
+    #
+    # @raise [RdkafkaError] When the operation failed
+    # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
+    #
+    # @return [Object] Operation-specific result
+    def wait(max_wait_timeout: 60, wait_timeout: 0.1)
+      timeout = if max_wait_timeout
+                  CURRENT_TIME.call + max_wait_timeout
+                else
+                  nil
+                end
+      loop do
+        if pending?
+          if timeout && timeout <= CURRENT_TIME.call
+            raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
+          end
+          sleep wait_timeout
+        elsif self[:response] != 0
+          raise_error
+        else
+          return create_result
+        end
+      end
+    end
+
+    # @return [String] the name of the operation (e.g. "delivery")
+    def operation_name
+      raise "Must be implemented by subclass!"
+    end
+
+    # @return [Object] operation-specific result
+    def create_result
+      raise "Must be implemented by subclass!"
+    end
+
+    # Allow subclasses to override
+    def raise_error
+      raise RdkafkaError.new(self[:response])
+    end
+
+    # Error that is raised when waiting for the handle to complete
+    # takes longer than the specified timeout.
+    class WaitTimeoutError < RuntimeError; end
+  end
+end

data/lib/rdkafka/admin/create_topic_handle.rb

@@ -0,0 +1,27 @@
+module Rdkafka
+  class Admin
+    class CreateTopicHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "create topic"
+      end
+
+      # @return [Boolean] whether the create topic was successful
+      def create_result
+        CreateTopicReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_topic_report.rb

@@ -0,0 +1,22 @@
+module Rdkafka
+  class Admin
+    class CreateTopicReport
+      # Any error message generated from the CreateTopic
+      # @return [String]
+      attr_reader :error_string
+
+      # The name of the topic created
+      # @return [String]
+      attr_reader :result_name
+
+      def initialize(error_string, result_name)
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
+        end
+        if result_name != FFI::Pointer::NULL
+          @result_name = @result_name = result_name.read_string
+        end
+      end
+    end
+  end
+end