karafka-rdkafka 0.12.0 → 0.12.1.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb0e8bec19a86f44453d32057d22c3958b4220e1d710564360b369905fd57d81
-  data.tar.gz: 645b3ab638f51780de30a21e133bd07b0e80fd33a1a59e604ec1dc064fc2b9c4
+  metadata.gz: 144a8490b1603c1fb10739ebd7a395980f658452ab3e84cf4a9e283d51eb5d47
+  data.tar.gz: 290812855c089866860eaf03a543b1191971d9c1ef3865377a679f96d734dc22
 SHA512:
-  metadata.gz: 52120cbb353e7bf7af881f442fb7a36324a70b097969af2f0e855f99e820c0be88f26ab584a285a85408eda526a0e564e06420e8c4d8224e40ae7d00f70546ab
-  data.tar.gz: ba886f70d88433d6dce7247df1c6d3598d20642fd1a4c17cb045f0249f22b7264c329462e0e4ae986fa053dd2a7c3598b0379fe4924822484d60c9b6baaecc35
+  metadata.gz: 52bc8ad5183a7d472380d00b4a80bd5cf8893a34bed614b67c7c2a44f3ad51d6df4b7eec553b12a5e293c80dde31daa84bd0c59db370da46a464644ec23ed988
+  data.tar.gz: fdea9eed1b45f439c1e71a7fb4eb48f61f924953d45bde78291308f678814120a7b0feeb2ea31b0e3550f47df3936927c1303a40e7215a811a56b36ff00ff282

checksums.yaml.gz.sig

@@ -1,2 +1,2 @@
-��t“qiK����2]T��poy[��C�􀚝M~Z���{��ү�����)��j�ںUM���^k��%Qv4��K�Ul3o2`T���
-�o7s�;7��=�y��gol��u
-��B6��yĭ��Rw�y�Sdq=x�N�'a�0j��?���s�&K�
+��ڸ���IJ���C������o���<~T��ja�t��ۨ�<=(�K�\0�b����"�5.���ݹqc�cs�}��Y��"�7�t�����pH�2%�{	^$ �T��/v�n<�C6��
+}9�(����N��	��

data/.github/workflows/ci.yml

@@ -0,0 +1,53 @@
+name: ci
+
+concurrency: ci-${{ github.ref }}
+
+on:
+  pull_request:
+  push:
+  schedule:
+    - cron:  '0 1 * * *'
+
+env:
+  BUNDLE_RETRY: 6
+  BUNDLE_JOBS: 4
+
+jobs:
+  specs:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.2'
+          - '3.1'
+          - '3.1.0'
+          - '3.0'
+          - '3.0.0'
+          - '2.7'
+          - '2.7.0'
+          - '2.6.8'
+        include:
+          - ruby: '3.2'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install package dependencies
+        run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: true
+
+      - name: Run all specs
+        env:
+          GITHUB_COVERAGE: ${{matrix.coverage}}
+
+        run: |
+          bundle install --path vendor/bundle
+          cd ext && bundle exec rake && cd ..
+          docker-compose up -d --no-recreate
+          bundle exec rspec

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.12.1-beta1
+* Bumps librdkafka to 2.0.2 (lmaia)
+* Add support for adding more partitions via Admin API
+
 # 0.12.0
 * Bumps librdkafka to 1.9.0
 

data/certs/cert_chain.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
+YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
+MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
+jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
+F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
+ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
+lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
+rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
+KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
+Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
+cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
+iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
+G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
+vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
+yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
+EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
+Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
+KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
+Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
+MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+-----END CERTIFICATE-----

data/certs/karafka-pro.pem

@@ -0,0 +1,11 @@
+-----BEGIN RSA PUBLIC KEY-----
+MIIBigKCAYEApcd6ybskiNs9WUvBGVUE8GdWDehjZ9TyjSj/fDl/UcMYqY0R5YX9
+tnYxEwZZRMdVltKWxr88Qmshh1IQz6CpJVbcfYjt/158pSGPm+AUua6tkLqIvZDM
+ocFOMafmroI+BMuL+Zu5QH7HC2tkT16jclGYfMQkJjXVUQTk2UZr+94+8RlUz/CH
+Y6hPA7xPgIyPfyPCxz1VWzAwXwT++NCJQPBr5MqT84LNSEzUSlR9pFNShf3UCUT+
+8LWOvjFSNGmMMSsbo2T7/+dz9/FM02YG00EO0x04qteggwcaEYLFrigDN6/fM0ih
+BXZILnMUqC/qrfW2YFg4ZqKZJuxaALqqkPxrkBDYqoqcAloqn36jBSke6tc/2I/J
+2Afq3r53UoAbUH7h5I/L8YeaiA4MYjAuq724lHlrOmIr4D6yjYC0a1LGlPjLk869
+2nsVXNgomhVb071E6amR+rJJnfvkdZgCmEBFnqnBV5A1u4qgNsa2rVcD+gJRvb2T
+aQtjlQWKPx5xAgMBAAE=
+-----END RSA PUBLIC KEY-----

data/{rdkafka.gemspec → karafka-rdkafka.gemspec}

@@ -6,7 +6,6 @@ Gem::Specification.new do |gem|
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
   gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
   gem.license = 'MIT'
-  gem.homepage = 'https://github.com/thijsc/rdkafka-ruby'
 
   gem.files = `git ls-files`.split($\)
   gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -16,13 +15,12 @@ Gem::Specification.new do |gem|
   gem.version = Rdkafka::VERSION
   gem.required_ruby_version = '>= 2.6'
   gem.extensions = %w(ext/Rakefile)
+  gem.cert_chain = %w[certs/cert_chain.pem]
 
   if $PROGRAM_NAME.end_with?('gem')
     gem.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end
 
-  gem.cert_chain    = %w[certs/cert_chain.pem]
-
   gem.add_dependency 'ffi', '~> 1.15'
   gem.add_dependency 'mini_portile2', '~> 2.6'
   gem.add_dependency 'rake', '> 12'
@@ -33,4 +31,14 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'simplecov'
   gem.add_development_dependency 'guard'
   gem.add_development_dependency 'guard-rspec'
+
+  gem.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/karafka-rdkafka/blob/master/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/karafka-rdkafka/issues',
+    'source_code_uri' => 'https://github.com/karafka/karafka-rdkafka',
+    'documentation_uri' => 'https://karafka.io/docs',
+    'rubygems_mfa_required' => 'true'
+  }
 end

data/lib/rdkafka/admin/create_partitions_handle.rb

@@ -0,0 +1,27 @@
+module Rdkafka
+  class Admin
+    class CreatePartitionsHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "create partitions"
+      end
+
+      # @return [Boolean] whether the create topic was successful
+      def create_result
+        CreatePartitionsReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: CreatePartitionsReport.new(self[:error_string], self[:result_name]).error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_partitions_report.rb

@@ -0,0 +1,6 @@
+module Rdkafka
+  class Admin
+    class CreatePartitionsReport < CreateTopicReport
+    end
+  end
+end

data/lib/rdkafka/admin.rb

@@ -151,5 +151,66 @@ module Rdkafka
 
       delete_topic_handle
     end
+
+    # Creates extra partitions for a given topic
+    #
+    # @param topic_name [String]
+    # @param partition_count [Integer] how many partitions we want to end up with for given topic
+    #
+    # @raise [ConfigError] When the partition count or replication factor are out of valid range
+    # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
+    # @raise [RdkafkaError] When the topic configuration is invalid
+    #
+    # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
+    def create_partitions(topic_name, partition_count)
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      new_partitions_ptr = Rdkafka::Bindings.rd_kafka_NewPartitions_new(
+        FFI::MemoryPointer.from_string(topic_name),
+        partition_count,
+        error_buffer,
+        256
+      )
+      if new_partitions_ptr.null?
+        raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+      end
+
+      pointer_array = [new_partitions_ptr]
+      topics_array_ptr = FFI::MemoryPointer.new(:pointer)
+      topics_array_ptr.write_array_of_pointer(pointer_array)
+
+      # Get a pointer to the queue that our request will be enqueued on
+      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      if queue_ptr.null?
+        Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
+        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+      end
+
+      # Create and register the handle we will return to the caller
+      create_partitions_handle = CreatePartitionsHandle.new
+      create_partitions_handle[:pending] = true
+      create_partitions_handle[:response] = -1
+      CreatePartitionsHandle.register(create_partitions_handle)
+      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
+      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
+
+      begin
+        Rdkafka::Bindings.rd_kafka_CreatePartitions(
+          @native_kafka,
+          topics_array_ptr,
+          1,
+          admin_options_ptr,
+          queue_ptr
+        )
+      rescue Exception
+        CreatePartitionsHandle.remove(create_partitions_handle.to_ptr.address)
+        raise
+      ensure
+        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+        Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
+      end
+
+      create_partitions_handle
+    end
   end
 end

data/lib/rdkafka/bindings.rb

@@ -286,6 +286,16 @@ module Rdkafka
     attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
     attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
 
+    # Create partitions
+    RD_KAFKA_ADMIN_OP_CREATEPARTITIONS = 3
+    RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT = 102
+
+    attach_function :rd_kafka_CreatePartitions, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
+    attach_function :rd_kafka_NewPartitions_new, %i[pointer size_t pointer size_t], :pointer
+    attach_function :rd_kafka_NewPartitions_destroy, [:pointer], :void
+    attach_function :rd_kafka_event_CreatePartitions_result, [:pointer], :pointer
+    attach_function :rd_kafka_CreatePartitions_result_topics, [:pointer, :pointer], :pointer
+
     # Background Queue and Callback
 
     attach_function :rd_kafka_queue_get_background, [:pointer], :pointer

data/lib/rdkafka/callbacks.rb

@@ -36,6 +36,8 @@ module Rdkafka
           process_create_topic(event_ptr)
         elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
           process_delete_topic(event_ptr)
+        elsif event_type == Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
+          process_create_partitions(event_ptr)
         end
       end
 
@@ -74,6 +76,23 @@ module Rdkafka
           delete_topic_handle[:pending] = false
         end
       end
+
+      def self.process_create_partitions(event_ptr)
+        create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
+
+        # Get the number of create topic results
+        pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+        create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
+        create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
+        create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
+          create_partitions_handle[:response] = create_partitions_results[0].result_error
+          create_partitions_handle[:error_string] = create_partitions_results[0].error_string
+          create_partitions_handle[:result_name] = create_partitions_results[0].result_name
+          create_partitions_handle[:pending] = false
+        end
+      end
     end
 
     # FFI Function used for Message Delivery callbacks

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.12.0"
-  LIBRDKAFKA_VERSION = "1.9.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "59b6088b69ca6cf278c3f9de5cd6b7f3fd604212cd1c59870bc531c54147e889"
+  VERSION = "0.12.1.beta1"
+  LIBRDKAFKA_VERSION = "2.0.2"
+  LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
 end

data/lib/rdkafka.rb

@@ -6,6 +6,8 @@ require "rdkafka/admin/create_topic_handle"
 require "rdkafka/admin/create_topic_report"
 require "rdkafka/admin/delete_topic_handle"
 require "rdkafka/admin/delete_topic_report"
+require "rdkafka/admin/create_partitions_handle"
+require "rdkafka/admin/create_partitions_report"
 require "rdkafka/bindings"
 require "rdkafka/callbacks"
 require "rdkafka/config"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.1.beta1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-01-11 00:00:00.000000000 Z
+date: 2023-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -172,9 +172,9 @@ extensions:
 - ext/Rakefile
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
-- ".semaphore/semaphore.yml"
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
@@ -183,12 +183,17 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- certs/cert_chain.pem
+- certs/karafka-pro.pem
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
+- karafka-rdkafka.gemspec
 - lib/rdkafka.rb
 - lib/rdkafka/abstract_handle.rb
 - lib/rdkafka/admin.rb
+- lib/rdkafka/admin/create_partitions_handle.rb
+- lib/rdkafka/admin/create_partitions_report.rb
 - lib/rdkafka/admin/create_topic_handle.rb
 - lib/rdkafka/admin/create_topic_report.rb
 - lib/rdkafka/admin/delete_topic_handle.rb
@@ -208,7 +213,6 @@ files:
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
-- rdkafka.gemspec
 - spec/rdkafka/abstract_handle_spec.rb
 - spec/rdkafka/admin/create_topic_handle_spec.rb
 - spec/rdkafka/admin/create_topic_report_spec.rb
@@ -229,10 +233,17 @@ files:
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/thijsc/rdkafka-ruby
+homepage: 
 licenses:
 - MIT
-metadata: {}
+metadata:
+  funding_uri: https://karafka.io/#become-pro
+  homepage_uri: https://karafka.io
+  changelog_uri: https://github.com/karafka/karafka-rdkafka/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/karafka/karafka-rdkafka/issues
+  source_code_uri: https://github.com/karafka/karafka-rdkafka
+  documentation_uri: https://karafka.io/docs
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -244,11 +255,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.

data/.semaphore/semaphore.yml

@@ -1,23 +0,0 @@
-version: v1.0
-name: Rdkafka Ruby
-
-agent:
-  machine:
-    type: e1-standard-4
-    os_image: ubuntu1804
-
-blocks:
-  - name: Run specs
-    task:
-      jobs:
-        - name: bundle exec rspec
-          matrix:
-            - env_var: RUBY_VERSION
-              values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
-          commands:
-            - sem-version ruby $RUBY_VERSION
-            - checkout
-            - bundle install --path vendor/bundle
-            - cd ext && bundle exec rake && cd ..
-            - docker-compose up -d --no-recreate
-            - bundle exec rspec