bps-kafka 0.0.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c89e50cf3f915d9894d258b09ebcaa65a7ad4baf300c63449b9f6e1ac5d8222
-  data.tar.gz: 5ebf8065eb7ca3439442e3d40c22279bcf2d593e57699950e21f3899fc0dad17
+  metadata.gz: 332d68ff69826c69b3a32c7d343e7760f6eab8441a2264667138351e189b5a08
+  data.tar.gz: b5f753df0a51c938e2fdf0930d9f233b13fb268c2e1960809f2ffda3fda94fc7
 SHA512:
-  metadata.gz: fe48f42b65ab43a8e946fd021d2b31ada7e36412114beaa5b756c37992c8b261ff0b8e9eea8d3b97900b94090dc4e56ffaa69d20c4aee398ebb0a306d2682aff
-  data.tar.gz: c49f9cbf9e4b1e1999acabc9d499133ed298610cc05550408c739e4da0c44dc6f51d3dca6ab94ad7b2cdd24b30a538b3ea0c5831da87065c132cbc425a083dcf
+  metadata.gz: 8a524f30fc1066a99be4e33ac4ada7e4c3ca39c8a705d695589ebfea89d4162b9f0b0e5792150d087b6b28a7fe7de546f13402d62c599acdc27a130295e8f2b9
+  data.tar.gz: f6aded50c6d824e192ce981c36568793714c07b9b19bbb5f9347e424ba22abd1151e7ef927eb6713dc945cfd0d512efdef86b04f204772e56b48f863e52cca4d

data/lib/bps/kafka.rb

@@ -6,13 +6,11 @@ require 'bps/publisher/kafka_async'
 module BPS
   module Publisher
     register('kafka+sync') do |url, **opts|
-      addrs = CGI.unescape(url.host).split(',')
-      Kafka.new(addrs, **Kafka.coercer.coerce(opts))
+      Kafka.new(url, **Kafka.coercer.coerce(opts))
     end
 
     register('kafka') do |url, **opts|
-      addrs = CGI.unescape(url.host).split(',')
-      KafkaAsync.new(addrs, **Kafka.coercer.coerce(opts))
+      KafkaAsync.new(url, **Kafka.coercer.coerce(opts))
     end
   end
 end

data/lib/bps/publisher/kafka.rb

@@ -3,8 +3,10 @@ require 'bps/kafka'
 module BPS
   module Publisher
     class Kafka < Abstract
-      class Topic
+      class Topic < Abstract::Topic
         def initialize(producer, topic)
+          super()
+
           @producer = producer
           @topic = topic
         end
@@ -14,7 +16,7 @@ module BPS
           after_publish
         end
 
-        def flush
+        def flush(**)
           @producer.deliver_messages
         end
 
@@ -74,12 +76,15 @@ module BPS
         @coercer ||= BPS::Coercer.new(CLIENT_OPTS.merge(PRODUCER_OPTS)).freeze
       end
 
-      # @param [Array<String>] brokers the seed broker addresses.
+      # @param [Array<String>,URI] brokers the seed broker addresses.
       # @param [Hash] opts the options.
       # @see https://www.rubydoc.info/gems/ruby-kafka/Kafka/Client#initialize-instance_method
       def initialize(broker_addrs, **opts)
-        @topics   = {}
-        @client   = ::Kafka.new(broker_addrs, **opts.slice(*CLIENT_OPTS.keys))
+        super()
+
+        broker_addrs = parse_url(broker_addrs) if broker_addrs.is_a?(URI)
+        @topics = {}
+        @client = ::Kafka.new(broker_addrs, **opts.slice(*CLIENT_OPTS.keys))
         @producer = init_producer(**opts.slice(*PRODUCER_OPTS.keys))
       end
 
@@ -94,6 +99,14 @@ module BPS
 
       private
 
+      def parse_url(url)
+        port = url.port&.to_s || '9092'
+        CGI.unescape(url.host).split(',').map do |addr|
+          addr << ':' << port unless /:\d+$/.match?(addr)
+          addr
+        end
+      end
+
       def init_producer(**opts)
         @producer = @client.producer(**opts)
       end

data/lib/bps/publisher/kafka_async.rb

@@ -14,18 +14,18 @@ module BPS
       # @param [Hash] opts the options.
       # @option opts [Integer] :max_queue_size (defaults to: 1000)
       #                        the maximum number of messages allowed in the queue.
-      # @option opts [Integer] :delivery_threshold (defaults to: 0)
+      # @option opts [Integer] :delivery_threshold (defaults to: 1000)
       #                        if greater than zero, the number of buffered messages that will automatically
       #                        trigger a delivery.
-      # @option opts [Integer] :delivery_interval (defaults to: 0) if greater than zero, the number of
+      # @option opts [Integer] :delivery_interval (defaults to: 30) if greater than zero, the number of
       #                        seconds between automatic message deliveries.
-      def initialize(broker_addrs, **opts)
+      def initialize(broker_addrs, **opts) # rubocop:disable Lint/UselessMethodDefinition
         super
       end
 
       private
 
-      def init_producer(max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0)
+      def init_producer(max_queue_size: 1000, delivery_threshold: 1000, delivery_interval: 30)
         @client.async_producer(
           max_queue_size: max_queue_size,
           delivery_threshold: delivery_threshold,

data/spec/bps/kafka_spec.rb

@@ -1,38 +1,64 @@
 require 'spec_helper'
 require 'bps/kafka'
 
-def kafka_addrs
-  ENV.fetch('KAFKA_ADDRS', '127.0.0.1:9092').split(',').freeze
-end
+RSpec.describe 'Kafka', kafka: true do
+  context 'with addr resolving' do
+    let(:client) { instance_double(::Kafka::Client, producer: nil) }
 
-run_spec = \
-  begin
-    ::Kafka.new(kafka_addrs).brokers
-    true
-  rescue StandardError => e
-    warn "WARNING: unable to run #{File.basename __FILE__}: #{e.message}"
-    false
-  end
+    before       { allow(::Kafka).to receive(:new).and_return(client) }
 
-helper = proc do
-  def read_messages(topic_name, num_messages)
-    client = ::Kafka.new(kafka_addrs)
-    Enumerator.new do |y|
-      client.each_message(topic: topic_name, start_from_beginning: true) do |msg|
-        y << msg.value
+    it 'resolves simple URLs' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        allow(::Kafka).to receive(:new).with(['test.host:9092'], {}).and_return(client)
+      else
+        allow(::Kafka).to receive(:new).with(['test.host:9092']).and_return(client)
       end
-    end.take(num_messages)
-  ensure
-    client&.close
-  end
-end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://test.host:9092'))
+    end
+
+    it 'resolves URLs with multiple hosts' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        allow(::Kafka).to receive(:new).with(['foo.host:9092', 'bar.host:9092'], {}).and_return(client)
+      else
+        allow(::Kafka).to receive(:new).with(['foo.host:9092', 'bar.host:9092']).and_return(client)
+      end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://foo.host,bar.host:9092'))
+    end
 
-RSpec.describe 'Kafka', if: run_spec do
-  context BPS::Publisher::Kafka do
-    it_behaves_like 'publisher', url: "kafka+sync://#{kafka_addrs.join(',')}/", &helper
+    it 'resolves URLs with multiple hosts/ports' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        allow(::Kafka).to receive(:new).with(['foo.host:9093', 'bar.host:9092'], {}).and_return(client)
+      else
+        allow(::Kafka).to receive(:new).with(['foo.host:9093', 'bar.host:9092']).and_return(client)
+      end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://foo.host%3A9093,bar.host'))
+    end
   end
 
-  context BPS::Publisher::KafkaAsync do
-    it_behaves_like 'publisher', url: "kafka://#{kafka_addrs.join(',')}/", &helper
+  describe 'publishers' do
+    let(:kafka_addrs) { ENV.fetch('KAFKA_ADDRS', '127.0.0.1:9092').split(',') }
+
+    def read_messages(topic_name, num_messages)
+      client = ::Kafka.new(kafka_addrs)
+      Enumerator.new do |y|
+        client.each_message(topic: topic_name, start_from_beginning: true) do |msg|
+          y << msg.value
+        end
+      end.take(num_messages)
+    ensure
+      client&.close
+    end
+
+    context BPS::Publisher::Kafka do
+      let(:publisher_url) { "kafka+sync://#{CGI.escape(kafka_addrs.join(','))}/" }
+
+      it_behaves_like 'publisher'
+    end
+
+    context BPS::Publisher::KafkaAsync do
+      let(:publisher_url) { "kafka://#{CGI.escape(kafka_addrs.join(','))}/" }
+
+      it_behaves_like 'publisher'
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bps-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Black Square Media
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-01 00:00:00.000000000 Z
+date: 2021-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bps
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement