rdkafka 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c019ec1ebf89bf81ce40340c8866e589d86826e3
+  data.tar.gz: cf1fdaadf5527542dbc1847587f43d8497d7f3d6
+SHA512:
+  metadata.gz: b58a95c81f0ae2b09d347acf12dbaf71057c4c26c25a6411eb5212723eee33d0b107db61fef385a39bb05079db586ba2d9406f6f1f097f5bb49e8c6852206a33
+  data.tar.gz: 3f5b7bd2277397df7acc096b3145a2e65e33df4d9ee1c7646b5d33d20992daecdd78fffb996abaccbb6b3304d131a7123a8fb94b2d7fdef4e8ad43b6403cdac5

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+ext/ports
+ext/tmp
+*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Thijs Cadier
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,8 @@
+# Modern Kafka client library for Ruby based on librdkafka
+
+Kafka client library wrapping `librdkafka` using the FFI gem for Kafka 0.10+ and Ruby 2.1+".
+
+## Development
+
+Run `bundle` and `cd ext && bundle exec rake compile && cd ..`. You can then run
+`bundle exec rspec` to run the tests.

data/ext/Rakefile

@@ -0,0 +1,15 @@
+require File.expand_path('../../lib/rdkafka/version', __FILE__)
+require "mini_portile2"
+require "fileutils"
+
+task :default do
+  recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+  recipe.files = ["https://github.com/edenhill/librdkafka/archive/v#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz"]
+  recipe.configure_options = ["--host=#{recipe.host}"]
+  recipe.cook
+end
+
+task :clean do
+  FileUtils.rm_rf "ports"
+  FileUtils.rm_rf "tmp"
+end

data/lib/rdkafka.rb

@@ -0,0 +1,7 @@
+require "rdkafka/version"
+
+require "rdkafka/config"
+require "rdkafka/consumer"
+require "rdkafka/error"
+require "rdkafka/ffi"
+require "rdkafka/producer"

data/lib/rdkafka/config.rb

@@ -0,0 +1,78 @@
+module Rdkafka
+  class Config
+    DEFAULT_CONFIG = {
+      "api.version.request" => "true"
+    }
+
+    def initialize(config_hash = {})
+      @config_hash = DEFAULT_CONFIG.merge(config_hash)
+    end
+
+    def []=(key, value)
+      @config_hash[key] = value
+    end
+
+    def [](key)
+      @config_hash[key]
+    end
+
+    def consumer
+      Rdkafka::Consumer.new(native_kafka(native_config, :rd_kafka_consumer))
+    end
+
+    def producer
+      # Create Kafka config
+      config = native_config
+      # Set callback to receive delivery reports on config
+      Rdkafka::FFI.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::FFI::DeliveryCallback)
+      # Return producer with Kafka client
+      Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer))
+    end
+
+    class ConfigError < RuntimeError; end
+    class ClientCreationError < RuntimeError; end
+
+    private
+
+    # This method is only intented to be used to create a client,
+    # using it in another way will leak memory.
+    def native_config
+      config = Rdkafka::FFI.rd_kafka_conf_new
+
+      @config_hash.each do |key, value|
+        error_buffer = ::FFI::MemoryPointer.from_string(" " * 256)
+        result = Rdkafka::FFI.rd_kafka_conf_set(
+          config,
+          key,
+          value,
+          error_buffer,
+          256
+        )
+        unless result == :config_ok
+          raise ConfigError.new(error_buffer.read_string)
+        end
+      end
+
+      config
+    end
+
+    def native_kafka(config, type)
+      error_buffer = ::FFI::MemoryPointer.from_string(" " * 256)
+      handle = Rdkafka::FFI.rd_kafka_new(
+        type,
+        config,
+        error_buffer,
+        256
+      )
+
+      if handle.nil?
+        raise ClientCreationError.new(error_buffer.read_string)
+      end
+
+      ::FFI::AutoPointer.new(
+        handle,
+        Rdkafka::FFI.method(:rd_kafka_destroy)
+      )
+    end
+  end
+end

data/lib/rdkafka/consumer.rb

@@ -0,0 +1,7 @@
+module Rdkafka
+  class Consumer
+    def initialize(native_kafka)
+      @native_kafka = native_kafka
+    end
+  end
+end

data/lib/rdkafka/error.rb

@@ -0,0 +1,25 @@
+module Rdkafka
+  class RdkafkaError < RuntimeError
+    attr_reader :rdkafka_response
+
+    def initialize(response)
+      @rdkafka_response = response
+    end
+
+    def code
+      if @rdkafka_response.nil?
+        :unknown_error
+      else
+        Rdkafka::FFI.rd_kafka_err2name(@rdkafka_response).downcase.to_sym
+      end
+    end
+
+    def to_s
+      if @rdkafka_response.nil?
+        "Unknown error: Response code is nil"
+      else
+        Rdkafka::FFI.rd_kafka_err2str(@rdkafka_response)
+      end
+    end
+  end
+end

data/lib/rdkafka/ffi.rb

@@ -0,0 +1,79 @@
+require "ffi"
+require "mini_portile2"
+
+module Rdkafka
+  module FFI
+    extend ::FFI::Library
+    ffi_lib "ext/ports/#{MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION).host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.dylib"
+
+    # Polling
+    attach_function :rd_kafka_poll, [:pointer, :int], :void
+
+    # Message struct
+
+    class Message < ::FFI::Struct
+      layout :err, :int,
+             :rkt, :pointer,
+             :partition, :int32,
+             :payload, :pointer,
+             :len, :size_t,
+             :key, :pointer,
+             :key_len, :size_t,
+             :offset, :int64,
+             :_private, :pointer
+    end
+
+    # Errors
+
+    attach_function :rd_kafka_err2name, [:int], :string
+    attach_function :rd_kafka_err2str, [:int], :string
+
+    # Configuration
+
+    enum :kafka_config_response, [
+      :config_unknown, -2,
+      :config_invalid, -1,
+      :config_ok, 0
+    ]
+
+    attach_function :rd_kafka_conf_new, [], :pointer
+    attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
+
+    # Handle
+
+    enum :kafka_type, [
+      :rd_kafka_producer,
+      :rd_kafka_consumer
+    ]
+
+    attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
+    attach_function :rd_kafka_destroy, [:pointer], :void
+
+    # Producing
+
+    RD_KAFKA_VTYPE_END = 0
+    RD_KAFKA_VTYPE_TOPIC = 1
+    RD_KAFKA_VTYPE_RKT = 2
+    RD_KAFKA_VTYPE_PARTITION = 3
+    RD_KAFKA_VTYPE_VALUE = 4
+    RD_KAFKA_VTYPE_KEY = 5
+    RD_KAFKA_VTYPE_OPAQUE = 6
+    RD_KAFKA_VTYPE_MSGFLAGS = 7
+    RD_KAFKA_VTYPE_TIMESTAMP = 8
+
+    RD_KAFKA_MSG_F_COPY = 0x2
+
+    attach_function :rd_kafka_producev, [:pointer, :varargs], :int
+    callback :delivery_cb, [:pointer, :pointer, :pointer], :void
+    attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
+
+    DeliveryCallback = Proc.new do |client_ptr, message_ptr, opaque_ptr|
+      message = Message.new(message_ptr)
+      delivery_handle = Rdkafka::DeliveryHandle.new(message[:_private])
+      delivery_handle[:pending] = false
+      delivery_handle[:response] = message[:err]
+      delivery_handle[:partition] = message[:partition]
+      delivery_handle[:offset] = message[:offset]
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -0,0 +1,98 @@
+module Rdkafka
+  class Producer
+    def initialize(native_kafka)
+      @native_kafka = native_kafka
+      # Start thread to poll client for delivery callbacks
+      @thread = Thread.new do
+        loop do
+          Rdkafka::FFI.rd_kafka_poll(@native_kafka, 100)
+        end
+      end.abort_on_exception = true
+    end
+
+    def produce(topic:, payload: nil, key: nil, partition: nil, timestamp: nil)
+      # Start by checking and converting the input
+
+      # Get payload length
+      payload_size = if payload.nil?
+                       0
+                     else
+                       payload.bytesize
+                     end
+
+      # Get key length
+      key_size = if key.nil?
+                   0
+                 else
+                   key.bytesize
+                 end
+
+      # If partition is nil use -1 to let Kafka set the partition based
+      # on the key/randomly if there is no key
+      partition = -1 if partition.nil?
+
+      # If timestamp is nil use 0 and let Kafka set one
+      timestamp = 0 if timestamp.nil?
+
+      delivery_handle = DeliveryHandle.new
+      delivery_handle[:pending] = true
+      delivery_handle[:response] = 0
+      delivery_handle[:partition] = 0
+      delivery_handle[:offset] = 0
+
+      # Produce the message
+      response = Rdkafka::FFI.rd_kafka_producev(
+        @native_kafka,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_TOPIC, :string, topic,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::FFI::RD_KAFKA_MSG_F_COPY,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_TIMESTAMP, :int64, timestamp,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
+        :int, Rdkafka::FFI::RD_KAFKA_VTYPE_END
+      )
+
+      # Raise error if the produce call was not successfull
+      if response != 0
+        raise RdkafkaError.new(response)
+      end
+
+      delivery_handle
+    end
+  end
+
+  class DeliveryHandle < ::FFI::Struct
+    layout :pending, :bool,
+           :response, :int,
+           :partition, :int,
+           :offset, :int64
+
+    def pending?
+      self[:pending]
+    end
+
+    # Wait for the delivery report
+    def wait
+      loop do
+        if pending?
+          sleep 0.05
+          next
+        elsif self[:response] != 0
+          raise RdkafkaError.new(self[:response])
+        else
+          return DeliveryReport.new(self[:partition], self[:offset])
+        end
+      end
+    end
+  end
+
+  class DeliveryReport
+    attr_reader :partition, :offset
+
+    def initialize(partition, offset)
+      @partition = partition
+      @offset = offset
+    end
+  end
+end

data/lib/rdkafka/version.rb

@@ -0,0 +1,4 @@
+module Rdkafka
+  VERSION = "0.0.1"
+  LIBRDKAFKA_VERSION = "0.11.0"
+end

data/rdkafka.gemspec

@@ -0,0 +1,26 @@
+require File.expand_path('../lib/rdkafka/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors = ['Thijs Cadier']
+  gem.email = ["thijs@appsignal.com"]
+  gem.description = "Modern Kafka client library for Ruby based on librdkafka"
+  gem.summary = "Kafka client library wrapping librdkafka using the ffi gem and futures from concurrent-ruby for Kafka 0.10+"
+  gem.license = 'MIT'
+  gem.homepage = 'https://github.com/thijsc/rdkafka-ruby'
+
+  gem.files = `git ls-files`.split($\)
+  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name = 'rdkafka'
+  gem.require_paths = ['lib']
+  gem.version = Rdkafka::VERSION
+  gem.required_ruby_version = '>= 2.0'
+  gem.extensions = %w(ext/Rakefile)
+
+  gem.add_dependency 'ffi', '~> 1.9'
+  gem.add_dependency 'mini_portile2', '~> 2.2'
+
+  gem.add_development_dependency 'pry', '~> 0.10'
+  gem.add_development_dependency 'rspec', '~> 3.5'
+  gem.add_development_dependency 'rake', '~> 12.0'
+end

data/spec/rdkafka/config_spec.rb

@@ -0,0 +1,36 @@
+require "spec_helper"
+
+describe Rdkafka::Config do
+  it "should store configuration" do
+    config = Rdkafka::Config.new
+    config['key'] = 'value'
+    expect(config['key']).to eq 'value'
+  end
+
+  it "should use default configuration" do
+    config = Rdkafka::Config.new
+    expect(config['api.version.request']).to eq 'true'
+  end
+
+  it "should create a consumer with valid config" do
+    expect(rdkafka_config.consumer).to be_a Rdkafka::Consumer
+  end
+
+  it "should raise an error when creating a consumer with invalid config" do
+    config = Rdkafka::Config.new('invalid.key' => 'value')
+    expect {
+      config.consumer
+    }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
+  end
+
+  it "should create a producer with valid config" do
+    expect(rdkafka_config.producer).to be_a Rdkafka::Producer
+  end
+
+  it "should raise an error when creating a producer with invalid config" do
+    config = Rdkafka::Config.new('invalid.key' => 'value')
+    expect {
+      config.producer
+    }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
+  end
+end

data/spec/rdkafka/consumer_spec.rb

@@ -0,0 +1,5 @@
+require "spec_helper"
+
+describe Rdkafka::Consumer do
+
+end

data/spec/rdkafka/error_spec.rb

@@ -0,0 +1,31 @@
+require "spec_helper"
+
+describe Rdkafka::RdkafkaError do
+  describe "#code" do
+    it "should handle a nil response" do
+      expect(Rdkafka::RdkafkaError.new(nil).code).to eq :unknown_error
+    end
+
+    it "should handle an invalid response" do
+      expect(Rdkafka::RdkafkaError.new(933975).code).to eq :err_933975?
+    end
+
+    it "should return error messages from rdkafka" do
+      expect(Rdkafka::RdkafkaError.new(10).code).to eq :msg_size_too_large
+    end
+  end
+
+  describe "#to_s" do
+    it "should handle a nil response" do
+      expect(Rdkafka::RdkafkaError.new(nil).to_s).to eq "Unknown error: Response code is nil"
+    end
+
+    it "should handle an invalid response" do
+      expect(Rdkafka::RdkafkaError.new(933975).to_s).to eq "Err-933975?"
+    end
+
+    it "should return error messages from rdkafka" do
+      expect(Rdkafka::RdkafkaError.new(10).to_s).to eq "Broker: Message size too large"
+    end
+  end
+end

data/spec/rdkafka/ffi_spec.rb

@@ -0,0 +1,7 @@
+require "spec_helper"
+
+describe Rdkafka::FFI do
+  it "should successfully call librdkafka" do
+    Rdkafka::FFI.rd_kafka_conf_new
+  end
+end

data/spec/rdkafka/producer_spec.rb

@@ -0,0 +1,31 @@
+require "spec_helper"
+
+describe Rdkafka::Producer do
+  let(:producer) do
+    rdkafka_config.producer
+  end
+
+  it "should require a topic" do
+    expect {
+      producer.produce(
+        payload: "payload",
+        key: "key"
+     )
+    }.to raise_error ArgumentError, "missing keyword: topic"
+  end
+
+  it "should produce a message" do
+    handle = producer.produce(
+      topic: "produce_test_topic",
+      payload: "payload 1",
+      key: "key 1"
+    )
+    expect(handle.pending?).to be true
+
+    report = handle.wait
+    expect(handle.pending?).to be false
+    expect(report).not_to be_nil
+    expect(report.partition).to eq 0
+    expect(report.offset).to be > 0
+  end
+end

data/spec/rdkafka_spec.rb

@@ -0,0 +1,4 @@
+require "spec_helper"
+
+describe Rdkafka do
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require "pry"
+require "rspec"
+require "rdkafka"
+
+def rdkafka_config
+  Rdkafka::Config.new("bootstrap.servers" => "localhost:9092")
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: rdkafka
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Thijs Cadier
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: mini_portile2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+description: Modern Kafka client library for Ruby based on librdkafka
+email:
+- thijs@appsignal.com
+executables: []
+extensions:
+- ext/Rakefile
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- ext/Rakefile
+- lib/rdkafka.rb
+- lib/rdkafka/config.rb
+- lib/rdkafka/consumer.rb
+- lib/rdkafka/error.rb
+- lib/rdkafka/ffi.rb
+- lib/rdkafka/producer.rb
+- lib/rdkafka/version.rb
+- rdkafka.gemspec
+- spec/rdkafka/config_spec.rb
+- spec/rdkafka/consumer_spec.rb
+- spec/rdkafka/error_spec.rb
+- spec/rdkafka/ffi_spec.rb
+- spec/rdkafka/producer_spec.rb
+- spec/rdkafka_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/thijsc/rdkafka-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Kafka client library wrapping librdkafka using the ffi gem and futures from
+  concurrent-ruby for Kafka 0.10+
+test_files:
+- spec/rdkafka/config_spec.rb
+- spec/rdkafka/consumer_spec.rb
+- spec/rdkafka/error_spec.rb
+- spec/rdkafka/ffi_spec.rb
+- spec/rdkafka/producer_spec.rb
+- spec/rdkafka_spec.rb
+- spec/spec_helper.rb