kafka 0.5.0

data/examples/producer.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+
+require "kafka"
+require "securerandom"
+
+config = Kafka::Config.new({
+  "bootstrap.servers": "127.0.0.1:9092",
+})
+
+producer = Kafka::Producer.new(config)
+
+# Initialize a topic with 8 partitions and 1 replica per partition. This is
+# only for testing, a replication factor of 1 is not generally recommended for
+# production.
+admin = Kafka::Admin.new(config)
+admin.create_topic("ruby_test_topic", 8, 1)
+admin.close
+
+@run = true
+trap("INT")  { @run = false }
+trap("TERM") { @run = false }
+
+# Create several threads to publish messages to the topic. Producers are thread
+# safe and can be accessed from multiple threads.
+workers = 8.times.map do |i|
+  Thread.new do
+    while @run
+      producer.produce("ruby_test_topic", "#{i}: #{SecureRandom.uuid}") do |report|
+        # Wait for delivery confirmation from the cluster
+        report.wait
+        puts report.inspect
+      end
+
+      sleep(SecureRandom.rand % 0.2)
+    end
+  end
+end
+
+# Wait for all worker threads to finish
+workers.each(&:join)
+
+# Gracefully close the producer, flushing any remaining messages, and
+# processing and remaining callbacks.
+producer.close

data/ext/Rakefile

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative "../lib/kafka/version"
+
+require "mini_portile2"
+require "fileutils"
+
+desc "Compile librdkafka v#{Kafka::LIBRDKAFKA_VERSION}"
+task :default do
+  Rake::Task["build:release"].invoke(Kafka::LIBRDKAFKA_VERSION, Kafka::LIBRDKAFKA_CHECKSUM)
+end
+
+desc "Clean up and remove build artifacts"
+task :clean do
+  Dir.chdir(__dir__) do
+    FileUtils.rm_rf([
+      "tmp",
+      "ports",
+      Dir.glob("librdkafka.*"),
+    ])
+  end
+end
+
+namespace :build do
+  # Build librdkafka and store the library into ext/
+  #
+  # NOTE: To speed up the build try:
+  #   MAKE="make -j${nproc}" rake
+  def build(version:, checksum: nil)
+    recipe = MiniPortile.new("librdkafka", version)
+    recipe.files << {
+      url: "https://github.com/edenhill/librdkafka/archive/#{version}.tar.gz",
+      sha256: checksum,
+    }
+    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.cook
+
+    ext = recipe.host.include?("darwin") ? "dylib" : "so"
+    lib = File.join(recipe.path, "lib", "librdkafka.#{ext}")
+
+    # cp will copy the content following any symlinks
+    FileUtils.cp(lib, __dir__)
+
+    # Remove build directories
+    FileUtils.rm_rf([
+      File.join(__dir__, "tmp"),
+      File.join(__dir__, "ports"),
+    ])
+  end
+
+  desc "Download and build an official release of librdkafka"
+  task :release, [:version, :checksum] do |_task, args|
+    version  = args[:version]
+    checksum = args[:checksum]
+
+    raise ArgumentError, "version is required" if version.nil?
+    raise ArgumentError, "checksum is required" if checksum.nil?
+
+    # Prefix the version string to look like "v1.3.0" so the URL is correct.
+    version = version.start_with?("v") ? str : "v#{version}"
+
+    build(version: version, checksum: checksum)
+  end
+
+  desc "Build librdkafka at the given git sha or tag"
+  task :git, [:ref, :checksum] do |_task, args|
+    build(version: args[:ref], checksum: args[:checksum])
+  end
+end

data/kafka.gemspec

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "lib/kafka/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "kafka"
+  spec.version       = Kafka::VERSION
+  spec.authors       = ["Chris Gaffney"]
+  spec.email         = ["gaffneyc@gmail.com"]
+
+  spec.summary       = "Kafka client bindings to librdkafka"
+  spec.description   = <<~DESCRIPTION
+    Kafka provides binding to librdafka as well as a default producer and
+    consumer implementation.
+  DESCRIPTION
+
+  spec.homepage      = "http://github.com/deadmanssnitch/kafka"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "https://github.com/deadmanssnitch/kafka/blob/master/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.require_paths = ["lib"]
+  spec.extensions    = %w[ext/Rakefile]
+
+  spec.add_dependency "ffi"
+  spec.add_dependency "mini_portile2"
+
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/lib/kafka/admin.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi"
+
+module Kafka
+  # Admin provides a client for accessing the rdkafka Admin API to make changes
+  # to the cluster. The API provides was to create topics, delete topics, add
+  # new partitions for a topic, and manage configs
+  class Admin
+    # Create a new Admin client for accessing the librdkafka Admin API.
+    #
+    # @param config [Kafka::Config] Cluster config
+    def initialize(config = nil)
+      # Wrap a Producer since it appears to allocate the fewest resources.
+      @client = ::Kafka::FFI::Producer.new(config)
+    end
+
+    # Create a topic with the given name, number of partitions, and number of
+    # replicas per partition (replication factor). Total number of partitions
+    # will be partitions x replication_factor.
+    #
+    # @param name [String] Name of the topic to create
+    # @param partitions [Integer] Number of partitions the topic will have
+    # @param replication_factor [Integer] Number of replicas per partition to
+    #   have in the cluster.
+    #
+    # @param wait [Boolean] Wait up to timeout milliseconds for topic creation
+    #   to propogate to the cluster before returning.
+    # @param validate [Boolean] Only validate the request
+    # @param timeout [Integer] Time to wait in milliseconds for each operation
+    #   to complete. Total request execution time may be longer than timeout
+    #   due to multiple operations being done. Defaults to `socket.timeout.ms`
+    #   config setting.
+    #
+    # @return [nil] Create timed out
+    # @return [TopicResult] Response from the cluster with details about if the
+    #   topic was created or any errors.
+    def create_topic(name, partitions, replication_factor, wait: true, validate: false, timeout: nil)
+      req = ::Kafka::FFI::Admin::NewTopic.new(name, partitions, replication_factor)
+      opts = new_options(:create_topics, wait: wait, validate: validate, timeout: timeout)
+
+      res = @client.create_topics(req, options: opts)
+      if res
+        res[0]
+      end
+    ensure
+      opts.destroy
+      req.destroy
+    end
+
+    # Delete the topic with the given name
+    #
+    # @param name [String] Name of the topic to delete
+    #
+    # @param wait [Boolean] Wait up to timeout milliseconds for topic creation
+    #   to propogate to the cluster before returning.
+    # @param validate [Boolean] Only validate the request
+    # @param timeout [Integer] Time to wait in milliseconds for each operation
+    #   to complete. Total request execution time may be longer than timeout
+    #   due to multiple operations being done. Defaults to `socket.timeout.ms`
+    #   config setting.
+    #
+    # @return [nil] Delete timed out
+    # @return [TopicResult] Response from the cluster with details about the
+    #   deletion or any errors.
+    def delete_topic(name, wait: true, validate: false, timeout: nil)
+      req = ::Kafka::FFI::Admin::DeleteTopic.new(name)
+      opts = new_options(:create_topics, wait: wait, validate: validate, timeout: timeout)
+
+      res = @client.delete_topics(req, options: opts)
+      if res
+        res[0]
+      end
+    ensure
+      opts.destroy
+      req.destroy
+    end
+
+    # Get current config settings for the resource.
+    #
+    # @example Get configuration for a topic
+    #   describe_config(:topic, "events")
+    #
+    # @param type [:broker, :topic, :group] Type of resource
+    # @param name [String] Name of the resource
+    #
+    # @return [ConfigResource]
+    def describe_config(type, name, wait: true, validate: false, timeout: nil)
+      req = ::Kafka::FFI::Admin::ConfigResource.new(type, name)
+      opts = new_options(:create_topics, wait: wait, validate: validate, timeout: timeout)
+
+      res = @client.describe_configs(req, options: opts)
+      if res
+        res[0]
+      end
+    ensure
+      opts.destroy
+      req.destroy
+    end
+
+    # Retrieve metadata for the cluster
+    #
+    # @see Kafka::FFI::Client#metadata
+    #
+    # @return [Metadata]
+    def metadata(local_only: false, topic: nil, timeout: 1000)
+      @client.metadata(local_only: local_only, topic: topic, timeout: timeout)
+    end
+
+    # Destroy the Client, releasing all used resources back to the system. It
+    # is the application's responsbility to call #destroy when done with the
+    # client.
+    def destroy
+      @client.destroy
+    end
+    alias close destroy
+
+    private
+
+    def new_options(api, wait: false, timeout: nil, validate: false)
+      options = ::Kafka::FFI::Admin::AdminOptions.new(@client, api)
+
+      # Request timeout defaults to socket.timeout.ms unless set. We use the
+      # timeout for both request_timeout and operation timeout when not set. It
+      # simplifies the API even if it is a bad assumption.
+      if timeout.nil?
+        timeout = @client.config.get("socket.timeout.ms").to_i
+      end
+
+      options.set_request_timeout(timeout)
+      options.set_validate_only(validate)
+
+      if wait
+        options.set_operation_timeout(timeout)
+      end
+
+      options
+    end
+  end
+end

data/lib/kafka/config.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Kafka
+  class Config
+    # Create a new Config for initializing a Kafka Consumer or Producer. This
+    # config is reusable and can be used to configure multiple Consumers or
+    # Producers.
+    #
+    # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    #
+    # @param opts [Hash{[String, Symbol] => [String, Integer, nil, Boolean]}]
+    #
+    # @raise [TypeError] Value was not of the correct type
+    def initialize(opts = {})
+      @opts = {}
+      @callbacks = {}
+
+      # Use #set to rekey the options as strings and type check the value.
+      opts.each_pair do |key, val|
+        set(key, val)
+      end
+    end
+
+    # Retrieve the configured value for the key.
+    #
+    # @return [nil] Value is not set
+    # @return Configured value for the given key
+    def get(key)
+      @opts[key.to_s]
+    end
+
+    # Set configratuon option `key` to `value`.
+    #
+    # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    #
+    # @param key [#to_s] Configuration option
+    # @param value [String, Integer, Boolean, nil]
+    #
+    # @raise [TypeError] Value was not of the correct type
+    def set(key, val)
+      key = key.to_s
+
+      @opts[key] =
+        case val
+        when String, Integer, true, false, nil
+          val
+        else
+          raise TypeError, "#{key}'s value must be a String, Integer, true, or false"
+        end
+
+      nil
+    end
+
+    # Callback for the delivery status of a message published to the Kafka
+    # cluster.
+    #
+    # @note Producer only
+    #
+    # @see Kafka::FFI::Config#set_dr_msg_cb
+    def on_delivery_report(&block)
+      @callbacks[:delivery_report] = block
+    end
+
+    # @note Consumer only
+    #
+    # @see Kafka::FFI::Config#set_consume_cb
+    def on_consume(&block)
+      @callbacks[:consume] = block
+    end
+
+    # Callback for result of automatic or manual offset commits.
+    #
+    # @note Consumer only
+    #
+    # @see Kafka::FFI::Config#set_offset_commit_cb
+    def on_offset_commit(&block)
+      @callbacks[:offset_commit] = block
+    end
+
+    # Callback for errors from the cluster. Most errors are informational and
+    # should be ignored as librdkafka will attempt to recover. However fatal
+    # errors can be reported which should cause the system to gracefully
+    # shutdown.
+    #
+    # @see Kafka::FFI::Config#set_error_cb
+    def on_error(&block)
+      @callbacks[:error] = block
+    end
+
+    # Callback for when Brokers throttle a client
+    #
+    # @see Kafka::FFI::Config#set_throttle_cb
+    def on_throttle(&block)
+      @callbacks[:throttle] = block
+    end
+
+    # Callback for log messages
+    #
+    # @see Kafka::FFI::Config#set_log_cb
+    def on_log(&block)
+      @callbacks[:log] = block
+    end
+
+    # Callback for connetion stats
+    #
+    # @see Kafka::FFI::Config#set_stats_cb
+    def on_stats(&block)
+      @callbacks[:stats] = block
+    end
+
+    # Allocate and configure a new Kafka::FFI::Config that mirrors this Config.
+    # The returned Kafka::FFI::Config should be either passed to initialize a
+    # new Client or eventually destroyed. Once passed to a Client, the Config
+    # is now owned by the Client and should not be modified or destroyed.
+    #
+    # @return [Kafka::FFI::Config]
+    def to_ffi
+      conf = Kafka::FFI.rd_kafka_conf_new
+
+      @opts.each do |name, value|
+        conf.set(name, value)
+      end
+
+      # Omitted callbacks:
+      #  - background_event - Requires lower level usage
+      #  - rebalance        - Requires knowing the rebalance semantics
+      #  - all socket       - Unknown need at this level
+      #  - ssl_cert_verify  - Currently not needed
+      #  - oauthbearer_token_refresh - Unable to test
+      @callbacks.each do |name, callback|
+        case name
+        when :delivery_report  then conf.set_dr_msg_cb(&callback)
+        when :consume          then conf.set_consume_cb(&callback)
+        when :offset_commit    then conf.set_offset_commit_cb(&callback)
+        when :error            then conf.set_error_cb(&callback)
+        when :throttle         then conf.set_throttle_cb(&callback)
+        when :log              then conf.set_log_cb(&callback)
+        when :stats            then conf.set_stats_cb(&callback)
+        end
+      end
+
+      conf
+    end
+  end
+end

data/lib/kafka/consumer.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "kafka/poller"
+
+module Kafka
+  class Consumer
+    # Returns the backing Kafka::FFI::Consumer.
+    #
+    # @DANGER Using the backing Consumer means being aware of memory management
+    #   and could leave the consumer in a bad state. Make sure you know what
+    #   you're doing.
+    #
+    # @return [Kafka::FFI::Consumer]
+    attr_reader :client
+
+    # @param config [Kafka::Config]
+    def initialize(config)
+      # Initialize the client
+      @client = Kafka::FFI::Consumer.new(config)
+
+      # Event loop polling for events so callbacks are fired.
+      @poller = Poller.new(@client)
+    end
+
+    # Subscribe the consumer to the given list of topics. Once the
+    # subscriptions have become active and partitions assigned, calls to #poll
+    # will yield messages for the subscribed topics.
+    #
+    # subscribe will _set_ the list of subscriptions, removing any that are not
+    # included in the most recent call.
+    #
+    # @param topic [String, Array<String>] Topics to subscribe to
+    def subscribe(topic, *rest)
+      @client.subscribe(topic, *rest)
+    end
+
+    # Retrieves the set of topic + partition assignments for the consumer.
+    #
+    # @example
+    #   consumer.assignment # => { "topic" => [1,2,3] }
+    #
+    # @return [Hash{String => Array<Integer>}] List of partition assignments
+    #   keyed by the topic name.
+    def assignments
+      @client.assignment
+    end
+
+    # Poll the consumer for waiting message.
+    #
+    # @param timeout [Integer] Time to wait in milliseconds for a message to be
+    #   available.
+    def poll(timeout: 250, &block)
+      @client.consumer_poll(timeout, &block)
+    end
+
+    # @param msg [Consumer::Message]
+    def commit(msg, async: false)
+      list = Kafka::FFI::TopicPartitionList.new(1)
+
+      list.add(msg.topic, msg.partition)
+      list.set_offset(msg.topic, msg.partition, msg.offset + 1)
+
+      @client.commit(list, async)
+    ensure
+      list.destroy
+    end
+
+    # Gracefully shutdown the consumer and it's connections.
+    #
+    # @note After calling #close it is unsafe to call any other method on the
+    #   Consumer.
+    def close
+      # @see https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#high-level-kafkaconsumer
+      @poller.stop
+
+      # Gracefully shutdown the consumer, leaving the consumer group,
+      # committing any remaining offsets, and releasing resources back to the
+      # system.
+      #
+      # This will effectively call #close on the Client automatically. Trying
+      # to follow the documentation and calling #close before #destroy caused
+      # warnings due to brokers disconnecting but just calling #destroy fixes
+      # that.
+      @client.destroy
+    end
+  end
+end

data/lib/kafka/error.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Kafka
+  class Error < StandardError; end
+
+  # ::Kafka::ResponseError is an Error that can be raised based on an :error_code as
+  # returned from the librdkafka API.
+  #
+  # @see rdkafka.h RD_KAFKA_RESP_ERR_*
+  # @see rdkafka.h rd_kafka_resp_err_t
+  class ::Kafka::ResponseError < Error
+    # @attr code [Integer] Error code as defined by librdkafka.
+    attr_reader :code
+
+    def initialize(code, message = nil)
+      @code = code
+      @message = message
+    end
+
+    # Returns the librdkafka error constant for this error.
+    # @return [String]
+    def name
+      "RD_KAFKA_RESP_ERR_#{::Kafka::FFI.rd_kafka_err2name(@code)}"
+    end
+
+    # Returns true when the error is from internal to librdkafka or false when
+    # the error was received from a broker or timeout.
+    #
+    # @see https://github.com/edenhill/librdkafka/blob/4818ecadee/src/rdkafka.h#L245
+    #
+    # @return [true] Error was internal to librdkafka
+    # @return [false] Error was returned by the cluster
+    def internal?
+      code < 0
+    end
+
+    # Returns a human readable error description
+    #
+    # @return [String] Human readable description of the error.
+    def to_s
+      @message || ::Kafka::FFI.rd_kafka_err2str(@code)
+    end
+  end
+end

data/lib/kafka/ffi/admin/admin_options.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "kafka/ffi/opaque_pointer"
+
+module Kafka::FFI::Admin
+  class AdminOptions < ::Kafka::FFI::OpaquePointer
+    def self.new(client, api)
+      ::Kafka::FFI.rd_kafka_AdminOptions_new(client, api)
+    end
+
+    # rubocop:disable Naming/AccessorMethodName
+
+    # Sets the overall request timeout which includes broker lookup, request
+    # transmissing, operation time, and response processing.
+    #
+    # Valid for all admin requests.
+    #
+    # @note Default request timeout is `socket.timeout.ms` config option.
+    #
+    # @param timeout [-1] Wait indefinitely for request to finish
+    # @param timeout [Integer] Time to wait in milliseconds for request to be
+    #   processed.
+    #
+    # @raise [ResponseError<RD_KAFKA_RESP_ERR__INVALID_ARG>] Timeout was out of
+    #   range.
+    def set_request_timeout(timeout)
+      error = ::FFI::MemoryPointer.new(:char, 512)
+
+      resp = ::Kafka::FFI.rd_kafka_AdminOptions_set_request_timeout(self, timeout, error, error.size)
+      if resp != :ok
+        raise ::Kafka::ResponseError.new(resp, error.read_string)
+      end
+
+      nil
+    ensure
+      error.free
+    end
+
+    # Set the broker's operation wait timeout for the request to be processed
+    # by the cluster.
+    #
+    # Only valid for :create_topics, :delete_topics, and :create_partitions
+    # operations.
+    #
+    # @param timeout [-1, 0] Return immediately after starting the operation.
+    # @param timeout [Integer] Max time to wait in milliseconds for the
+    #   operation to propogate to the cluster.
+    #
+    # @raise [ResponseError<RD_KAFKA_RESP_ERR__INVALID_ARG>] Timeout was out of
+    #   range.
+    def set_operation_timeout(timeout)
+      error = ::FFI::MemoryPointer.new(:char, 512)
+
+      resp = ::Kafka::FFI.rd_kafka_AdminOptions_set_operation_timeout(self, timeout, error, error.size)
+      if resp != :ok
+        raise ::Kafka::ResponseError.new(resp, error.read_string)
+      end
+
+      nil
+    ensure
+      error.free
+    end
+
+    # Tell the broker to only validate the request without actually performing
+    # the operation.
+    #
+    # Only valid for :create_topics, :delete_topics, and :create_partitions
+    # operations.
+    #
+    # @param on [Boolean] True to validate the request without performing it.
+    #
+    # @raise [Kafka::ResponseError]
+    def set_validate_only(on)
+      error = ::FFI::MemoryPointer.new(:char, 512)
+
+      resp = ::Kafka::FFI.rd_kafka_AdminOptions_set_validate_only(self, on, error, error.size)
+      if resp != :ok
+        raise ::Kafka::ResponseError.new(resp, error.read_string)
+      end
+
+      nil
+    ensure
+      error.free
+    end
+
+    # Override which broker the Admin request will be sent to. By default,
+    # requests are sent to the controller Broker with a couple exceptions (see
+    # librdkafka)
+    #
+    # @note This API shoudl typically not be used and primarily serves as a
+    #   workaround in some cases.
+    #
+    # @see rdkafka.h rd_kafka_AdminOptions_set_broker
+    #
+    # @param broker_id [Integer] ID of the Broker to receive the request.
+    def set_broker(broker_id)
+      error = ::FFI::MemoryPointer.new(:char, 512)
+
+      resp = ::Kafka::FFI.rd_kafka_AdminOptions_set_broker(self, broker_id, error, error.size)
+      if resp != :ok
+        raise ::Kafka::ResponseError.new(resp, error.read_string)
+      end
+
+      nil
+    ensure
+      error.free
+    end
+
+    # rubocop:enable Naming/AccessorMethodName
+
+    # Ruby like aliases for librdkafka functions
+    alias request_timeout= set_request_timeout
+    alias operation_timeout= set_operation_timeout
+    alias validate_only= set_validate_only
+    alias broker= set_broker
+
+    def destroy
+      ::Kafka::FFI.rd_kafka_AdminOptions_destroy(self)
+    end
+  end
+end