kafka 0.5.0

data/lib/kafka/producer/delivery_report.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+class Kafka::Producer
+  class DeliveryReport
+    # @return [nil] Delivery was successful or report has not been received
+    #   yet.
+    # @return [Kafka::ResponseError] Error delivering the Message.
+    attr_reader :error
+
+    # @return [nil] Report has not been received yet
+    # @return [String] Name of the topic Message was delivered to.
+    attr_reader :topic
+
+    # @return [nil] Report has not been received yet
+    # @return [Integer] Offset for the message on partition.
+    attr_reader :offset
+
+    # @return [nil] Report has not been received yet
+    # @return [Integer] Partition the message was delivered to.
+    attr_reader :partition
+
+    # Initializes a new DeliveryReport
+    #
+    # @param block [Proc] Callback to call with the DeliveryReport when it is
+    #   received from the cluster.
+    def initialize(&block)
+      @mutex = Mutex.new
+      @waiter = ConditionVariable.new
+
+      @error = nil
+      @topic = nil
+      @offset = nil
+      @partition = nil
+      @callback = block
+
+      # Will be set to true by a call to #done. Fast out for any callers to
+      # #wait that may come in after done has already been called.
+      @done = false
+    end
+
+    # Returns true when the report has been received back from the kafka
+    # cluster.
+    #
+    # @return [Boolean] True when the server has reported back on the
+    #   delivery.
+    def received?
+      @done
+    end
+
+    # @return [Boolean] Is the report for an error?
+    def error?
+      error.nil?
+    end
+
+    # Returns if the delivery was successful
+    #
+    # @return [Boolean] True when the report was delivered to the cluster
+    #   successfully.
+    def successful?
+      !error
+    end
+
+    # @private
+    #
+    # Set the response based on the message and notify anyone waiting on the
+    # result.
+    #
+    # @param message [Kafka::FFI::Message]
+    def done(message)
+      @mutex.synchronize do
+        @error = message.error
+
+        @offset = message.offset
+        @topic = message.topic
+        @partition = message.partition
+
+        @done = true
+        @waiter.broadcast
+
+        remove_instance_variable(:@mutex)
+        remove_instance_variable(:@waiter)
+      end
+
+      if @callback
+        @callback.call(self)
+      end
+    end
+
+    # Wait for a report to be received for the delivery from the cluster.
+    #
+    # @param timeout [Integer] Maximum time to wait in milliseconds.
+    #
+    # @raise [Kafka::ResponseError<RD_KAFKA_RESP_ERR__TIMED_OUT>] No report
+    #   received before timeout.
+    def wait(timeout: 5000)
+      # Fast out since the delivery report has already been reported back from
+      # the cluster.
+      if @done
+        return
+      end
+
+      @mutex.synchronize do
+        # Convert from milliseconds to seconds to match Ruby's API. Takes
+        # milliseconds to be consistent with librdkafka APIs.
+        if timeout
+          timeout /= 1000.0
+        end
+
+        @waiter.wait(@mutex, timeout)
+
+        # No report was received for the message before we timed out waiting.
+        if !@done
+          raise ::Kafka::ResponseError, ::Kafka::FFI::RD_KAFKA_RESP_ERR__TIMED_OUT
+        end
+      end
+
+      nil
+    end
+  end
+end

data/lib/kafka/producer.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "kafka/poller"
+
+module Kafka
+  class Producer
+    require "kafka/producer/delivery_report"
+
+    # Returns the backing Kafka::FFI::Producer.
+    #
+    # @DANGER Using the backing Producer means being aware of memory management
+    #   and could leave the producer in a bad state. Make sure you know what
+    #   you're doing.
+    #
+    # @return [Kafka::FFI::Producer]
+    attr_reader :client
+
+    # Initialize a new Producer for the configured cluster.
+    #
+    # @param config [Config]
+    def initialize(config)
+      config = config.dup
+
+      # Configure callbacks
+      config.on_delivery_report(&method(:on_delivery_report))
+
+      @client = ::Kafka::FFI::Producer.new(config)
+
+      # Periodically call poll on the client to ensure callbacks are fired.
+      @poller = Poller.new(@client)
+    end
+
+    # Produce and publish a message to the Kafka cluster.
+    #
+    # @param topic [String] Topic to publish the message to
+    # @param payload [String] Message's payload
+    # @param key [String] Optional partitioning key Kafka can use to determine
+    #   the correct partition.
+    # @param partition [-1, nil] Kafka will determine the partition
+    #   automatically based on the `partitioner` config option.
+    # @param partition [Integer] Specifiy partition to publish the message to.
+    # @param headers [Hash{String => String}] Set of headers to attach to the
+    #   message.
+    # @param timestamp [nil] Let Kafka automatically assign Message timestamp
+    # @param timestamp [Time] Timestamp for the message
+    # @param timestamp [Integer] Timestamp as milliseconds since unix epoch
+    #
+    # @param block [Proc] Optional asyncronous callback to be called when the
+    #   delivery status of the message is reported back from librdkafka. The
+    #   callback MUST avoid expensive or long running processing as that may
+    #   causes issues inside librdkafka.
+    #
+    # @yield [report] Called asyncronously when the report is received from
+    #   Kafka on the success or failure of the delivery.
+    # @yieldparam report [DeliveryReport] Delivery status of the message.
+    #
+    # @return [DeliveryReport] Report of the success or failure of the
+    #   delivery. Call #wait to block until the report is received.
+    def produce(topic, payload, key: nil, partition: nil, headers: nil, timestamp: nil, &block)
+      report = DeliveryReport.new(&block)
+
+      # Allocate a pointer to a small chunk of memory. We will use the pointer
+      # (not the value it points to) as a key for looking up the DeliveryReport
+      # in the callback.
+      #
+      # Using a MemoryPointer as a key also ensures we have a reference to the
+      # Pointer so it doesn't get garbage collected away and it can be freed it
+      # in the callback since the raw FFI::Pointer disallows #free as FFI
+      # doesn't believe we allocated it.
+      opaque = Kafka::FFI::Opaque.new(report)
+
+      @client.produce(topic, payload, key: key, partition: partition, headers: headers, timestamp: timestamp, opaque: opaque)
+
+      report
+    rescue
+      opaque.free
+
+      raise
+    end
+
+    # Wait until all outstanding produce requests are completed.
+    #
+    # @raise [Kafka::ResponseError] Timeout was reached before all
+    #   outstanding requests were completed.
+    def flush(timeout: 1000)
+      @client.flush(timeout: timeout)
+    end
+
+    # Gracefully shutdown the Producer, flushing any pending deliveries, and
+    # finally releasing an memory back to the system.
+    #
+    # @note Once #close is call it is no longer safe to call any other method
+    #   on the Producer.
+    #
+    # @param timeout [Integer] Maximum time to wait in milliseconds for
+    #   messages to be flushed.
+    def close(timeout: 30000)
+      # @see https://github.com/edenhill/librdkafka/blob/master/INTRODUCTION.md#producer
+      @poller.stop
+
+      @client.flush(timeout: timeout)
+      @client.poll
+
+      # Client handles destroying cached Topics
+      @client.destroy
+    end
+
+    private
+
+    # @param client [Kafka::FFI::Producer]
+    # @param message [Kafka::FFI::Message]
+    # @param opaque [FFI::Pointer]
+    def on_delivery_report(_client, message, _opaque)
+      op = message.opaque
+      if op.nil?
+        return
+      end
+
+      begin
+        report = op.value
+        report.done(message)
+      ensure
+        op.free
+      end
+    end
+  end
+end

data/lib/kafka/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Kafka
+  VERSION = "0.5.0"
+
+  LIBRDKAFKA_VERSION  = "1.3.0"
+  LIBRDKAFKA_CHECKSUM = "465cab533ebc5b9ca8d97c90ab69e0093460665ebaf38623209cf343653c76d2"
+end

data/lib/kafka.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Kafka
+  require "kafka/ffi"
+  require "kafka/admin"
+  require "kafka/error"
+  require "kafka/config"
+  require "kafka/version"
+  require "kafka/consumer"
+  require "kafka/producer"
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: kafka
+version: !ruby/object:Gem::Version
+  version: 0.5.0
+platform: ruby
+authors:
+- Chris Gaffney
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mini_portile2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  Kafka provides binding to librdafka as well as a default producer and
+  consumer implementation.
+email:
+- gaffneyc@gmail.com
+executables: []
+extensions:
+- ext/Rakefile
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/consumer.rb
+- examples/producer.rb
+- ext/Rakefile
+- kafka.gemspec
+- lib/kafka.rb
+- lib/kafka/admin.rb
+- lib/kafka/config.rb
+- lib/kafka/consumer.rb
+- lib/kafka/error.rb
+- lib/kafka/ffi.rb
+- lib/kafka/ffi/admin.rb
+- lib/kafka/ffi/admin/admin_options.rb
+- lib/kafka/ffi/admin/config_entry.rb
+- lib/kafka/ffi/admin/config_resource.rb
+- lib/kafka/ffi/admin/delete_topic.rb
+- lib/kafka/ffi/admin/new_partitions.rb
+- lib/kafka/ffi/admin/new_topic.rb
+- lib/kafka/ffi/admin/result.rb
+- lib/kafka/ffi/admin/topic_result.rb
+- lib/kafka/ffi/broker_metadata.rb
+- lib/kafka/ffi/client.rb
+- lib/kafka/ffi/config.rb
+- lib/kafka/ffi/consumer.rb
+- lib/kafka/ffi/error.rb
+- lib/kafka/ffi/event.rb
+- lib/kafka/ffi/group_info.rb
+- lib/kafka/ffi/group_list.rb
+- lib/kafka/ffi/group_member_info.rb
+- lib/kafka/ffi/message.rb
+- lib/kafka/ffi/message/header.rb
+- lib/kafka/ffi/metadata.rb
+- lib/kafka/ffi/opaque.rb
+- lib/kafka/ffi/opaque_pointer.rb
+- lib/kafka/ffi/partition_metadata.rb
+- lib/kafka/ffi/producer.rb
+- lib/kafka/ffi/queue.rb
+- lib/kafka/ffi/topic.rb
+- lib/kafka/ffi/topic_config.rb
+- lib/kafka/ffi/topic_metadata.rb
+- lib/kafka/ffi/topic_partition.rb
+- lib/kafka/ffi/topic_partition_list.rb
+- lib/kafka/poller.rb
+- lib/kafka/producer.rb
+- lib/kafka/producer/delivery_report.rb
+- lib/kafka/version.rb
+homepage: http://github.com/deadmanssnitch/kafka
+licenses:
+- MIT
+metadata:
+  homepage_uri: http://github.com/deadmanssnitch/kafka
+  source_code_uri: http://github.com/deadmanssnitch/kafka
+  changelog_uri: https://github.com/deadmanssnitch/kafka/blob/master/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Kafka client bindings to librdkafka
+test_files: []