bps-kafka 0.0.3 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68826145fba6a103c652c008207089c913a18999a41acdcc35082d526a8979b1
-  data.tar.gz: f14e9f89c8ff5d968b744483e381abb41f6b1265d6d696688a31439f0fdbf591
+  metadata.gz: 98dcf92fca51a26782f1ac6824503e60172860b0eddaa0ef076b5cc47e3cad0d
+  data.tar.gz: 447de9c24af68374ed0b5b8a50e24ea1713ba485ed76c2ab021290c71e806d73
 SHA512:
-  metadata.gz: b9b4d9ae93a101c874f0cd2f0a5f07b7b577e6526de8a3ccc978cd346344d768e354202aa6b4652589334e0a1c56d5a556c1053929b2e3c1a1152d9556911571
-  data.tar.gz: 52d62b30b81d2f18a1098a47eff1314406aef6b371293ca0ef9ae10b17c8fa1a53043e5c21d7490b10717255fbb9408fa1df67879eab36e947cf06fe27782500
+  metadata.gz: e56d669ed4fabdd2b8068ddc1d22d5efd2c6ccf0d6d05052303ba95eb798cf35549835fec2f6a959a50600cce076bd6965714f32366467475afe8cc44fe7daf5
+  data.tar.gz: 9c97c002fb7dbae88abea0c0572dd5e7c92a121a45c0da12098677f92296c193754b584c1a422e5a91a9052638826f3a14071b9af1fa1a6e5d44d85855b283b4

data/lib/bps/kafka.rb

@@ -6,13 +6,11 @@ require 'bps/publisher/kafka_async'
 module BPS
   module Publisher
     register('kafka+sync') do |url, **opts|
-      addrs = CGI.unescape(url.host).split(',')
-      Kafka.new(addrs, **Kafka.coercer.coerce(opts))
+      Kafka.new(url, **Kafka.coercer.coerce(opts))
     end
 
     register('kafka') do |url, **opts|
-      addrs = CGI.unescape(url.host).split(',')
-      KafkaAsync.new(addrs, **Kafka.coercer.coerce(opts))
+      KafkaAsync.new(url, **Kafka.coercer.coerce(opts))
     end
   end
 end

data/lib/bps/publisher/kafka.rb

@@ -76,14 +76,15 @@ module BPS
         @coercer ||= BPS::Coercer.new(CLIENT_OPTS.merge(PRODUCER_OPTS)).freeze
       end
 
-      # @param [Array<String>] brokers the seed broker addresses.
+      # @param [Array<String>,URI] brokers the seed broker addresses.
       # @param [Hash] opts the options.
       # @see https://www.rubydoc.info/gems/ruby-kafka/Kafka/Client#initialize-instance_method
       def initialize(broker_addrs, **opts)
         super()
 
-        @topics   = {}
-        @client   = ::Kafka.new(broker_addrs, **opts.slice(*CLIENT_OPTS.keys))
+        broker_addrs = parse_url(broker_addrs) if broker_addrs.is_a?(URI)
+        @topics = {}
+        @client = ::Kafka.new(broker_addrs, **opts.slice(*CLIENT_OPTS.keys))
         @producer = init_producer(**opts.slice(*PRODUCER_OPTS.keys))
       end
 
@@ -98,6 +99,14 @@ module BPS
 
       private
 
+      def parse_url(url)
+        port = url.port&.to_s || '9092'
+        CGI.unescape(url.host).split(',').map do |addr|
+          addr << ':' << port unless /:\d+$/.match?(addr)
+          addr
+        end
+      end
+
       def init_producer(**opts)
         @producer = @client.producer(**opts)
       end

data/lib/bps/publisher/kafka_async.rb

@@ -14,10 +14,10 @@ module BPS
       # @param [Hash] opts the options.
       # @option opts [Integer] :max_queue_size (defaults to: 1000)
       #                        the maximum number of messages allowed in the queue.
-      # @option opts [Integer] :delivery_threshold (defaults to: 0)
+      # @option opts [Integer] :delivery_threshold (defaults to: 1000)
       #                        if greater than zero, the number of buffered messages that will automatically
       #                        trigger a delivery.
-      # @option opts [Integer] :delivery_interval (defaults to: 0) if greater than zero, the number of
+      # @option opts [Integer] :delivery_interval (defaults to: 30) if greater than zero, the number of
       #                        seconds between automatic message deliveries.
       def initialize(broker_addrs, **opts) # rubocop:disable Lint/UselessMethodDefinition
         super
@@ -25,7 +25,7 @@ module BPS
 
       private
 
-      def init_producer(max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0)
+      def init_producer(max_queue_size: 1000, delivery_threshold: 1000, delivery_interval: 30)
         @client.async_producer(
           max_queue_size: max_queue_size,
           delivery_threshold: delivery_threshold,

data/spec/bps/kafka_spec.rb

@@ -1,38 +1,64 @@
 require 'spec_helper'
 require 'bps/kafka'
 
-def kafka_addrs
-  ENV.fetch('KAFKA_ADDRS', '127.0.0.1:9092').split(',').freeze
-end
+RSpec.describe 'Kafka', kafka: true do
+  context 'with addr resolving' do
+    let(:client) { instance_double(::Kafka::Client, producer: nil) }
 
-run_spec = \
-  begin
-    ::Kafka.new(kafka_addrs).brokers
-    true
-  rescue StandardError => e
-    warn "WARNING: unable to run #{File.basename __FILE__}: #{e.message}"
-    false
-  end
+    before       { allow(::Kafka).to receive(:new).and_return(client) }
 
-helper = proc do
-  def read_messages(topic_name, num_messages)
-    client = ::Kafka.new(kafka_addrs)
-    Enumerator.new do |y|
-      client.each_message(topic: topic_name, start_from_beginning: true) do |msg|
-        y << msg.value
+    it 'resolves simple URLs' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        allow(::Kafka).to receive(:new).with(['test.host:9092'], {}).and_return(client)
+      else
+        allow(::Kafka).to receive(:new).with(['test.host:9092']).and_return(client)
       end
-    end.take(num_messages)
-  ensure
-    client&.close
-  end
-end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://test.host:9092'))
+    end
+
+    it 'resolves URLs with multiple hosts' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        allow(::Kafka).to receive(:new).with(['foo.host:9092', 'bar.host:9092'], {}).and_return(client)
+      else
+        allow(::Kafka).to receive(:new).with(['foo.host:9092', 'bar.host:9092']).and_return(client)
+      end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://foo.host,bar.host:9092'))
+    end
 
-RSpec.describe 'Kafka', if: run_spec do
-  context BPS::Publisher::Kafka do
-    it_behaves_like 'publisher', url: "kafka+sync://#{kafka_addrs.join(',')}/", &helper
+    it 'resolves URLs with multiple hosts/ports' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        allow(::Kafka).to receive(:new).with(['foo.host:9093', 'bar.host:9092'], {}).and_return(client)
+      else
+        allow(::Kafka).to receive(:new).with(['foo.host:9093', 'bar.host:9092']).and_return(client)
+      end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://foo.host%3A9093,bar.host'))
+    end
   end
 
-  context BPS::Publisher::KafkaAsync do
-    it_behaves_like 'publisher', url: "kafka://#{kafka_addrs.join(',')}/", &helper
+  describe 'publishers' do
+    let(:kafka_addrs) { ENV.fetch('KAFKA_ADDRS', '127.0.0.1:9092').split(',') }
+
+    def read_messages(topic_name, num_messages)
+      client = ::Kafka.new(kafka_addrs)
+      Enumerator.new do |y|
+        client.each_message(topic: topic_name, start_from_beginning: true) do |msg|
+          y << msg.value
+        end
+      end.take(num_messages)
+    ensure
+      client&.close
+    end
+
+    context BPS::Publisher::Kafka do
+      let(:publisher_url) { "kafka+sync://#{CGI.escape(kafka_addrs.join(','))}/" }
+
+      it_behaves_like 'publisher'
+    end
+
+    context BPS::Publisher::KafkaAsync do
+      let(:publisher_url) { "kafka://#{CGI.escape(kafka_addrs.join(','))}/" }
+
+      it_behaves_like 'publisher'
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bps-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.2.1
 platform: ruby
 authors:
 - Black Square Media
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2021-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bps
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
@@ -69,7 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Kafka adapter for bps