bps-kafka 0.0.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c35677b45d9df97c7ab28ef3ce4e910d2edcd6def70caa235923b48ab9cd057
-  data.tar.gz: 5d4ee5dce8765945dd11a373e4d461e1c5aaf08ee32e77f7e40c64954d934f2e
+  metadata.gz: b5d05abc6c4a93507504408f2896d0b58bfd0e0a31b8cc1158880cd10c0f9f4d
+  data.tar.gz: dedbc0320b57dab92bdc9496dce28e8bbf0f00d49783fb63b6970adf49ca411e
 SHA512:
-  metadata.gz: be7fcda9c77f2f3bb4d29f49bb29ecce4362f0bd41405e6567b93f9ce4c81019e522d3d7bfecb81d52e67d468bbc8e807c562d35cac8a05f1cf01d03d0a7eb61
-  data.tar.gz: af85befa5ff134a58df21d247b648b72d2c81206ffe1613c92ae8270d43bcb3ba3d12e9adcd31f3832ff0c2966d04e2ae377f93d5a3c8ff348a360e70f0021a5
+  metadata.gz: 5aefa008139667df54d170f7f7ddbce4084136885dae21565f2734a71ce4016d3d633964076b9c8b4bcf44a24c0bcc0ad260fb5925eb334599018c7d4a3817b5
+  data.tar.gz: 1bc340118cd5b21d996f78f0705f8acd60f8d13a75c1e0576a60bc99f923015d0834a26932efa19a2b05a0f6f631be33395c62f89091ad6515ef0100ad9886e8

data/lib/bps/kafka.rb

@@ -6,13 +6,11 @@ require 'bps/publisher/kafka_async'
 module BPS
   module Publisher
     register('kafka+sync') do |url, **opts|
-      addrs = CGI.unescape(url.host).split(',')
-      Kafka.new(addrs, **Kafka.coercer.coerce(opts))
+      Kafka.new(url, **Kafka.coercer.coerce(opts))
     end
 
     register('kafka') do |url, **opts|
-      addrs = CGI.unescape(url.host).split(',')
-      KafkaAsync.new(addrs, **Kafka.coercer.coerce(opts))
+      KafkaAsync.new(url, **Kafka.coercer.coerce(opts))
     end
   end
 end

data/lib/bps/publisher/kafka.rb

@@ -3,8 +3,10 @@ require 'bps/kafka'
 module BPS
   module Publisher
     class Kafka < Abstract
-      class Topic
+      class Topic < Abstract::Topic
         def initialize(producer, topic)
+          super()
+
           @producer = producer
           @topic = topic
         end
@@ -14,7 +16,7 @@ module BPS
           after_publish
         end
 
-        def flush
+        def flush(**)
           @producer.deliver_messages
         end
 
@@ -74,12 +76,15 @@ module BPS
         @coercer ||= BPS::Coercer.new(CLIENT_OPTS.merge(PRODUCER_OPTS)).freeze
       end
 
-      # @param [Array<String>] brokers the seed broker addresses.
+      # @param [Array<String>,URI] brokers the seed broker addresses.
       # @param [Hash] opts the options.
       # @see https://www.rubydoc.info/gems/ruby-kafka/Kafka/Client#initialize-instance_method
       def initialize(broker_addrs, **opts)
-        @topics   = {}
-        @client   = ::Kafka.new(broker_addrs, **opts.slice(*CLIENT_OPTS.keys))
+        super()
+
+        broker_addrs = parse_url(broker_addrs) if broker_addrs.is_a?(URI)
+        @topics = {}
+        @client = ::Kafka.new(broker_addrs, **opts.slice(*CLIENT_OPTS.keys))
         @producer = init_producer(**opts.slice(*PRODUCER_OPTS.keys))
       end
 
@@ -94,6 +99,14 @@ module BPS
 
       private
 
+      def parse_url(url)
+        port = url.port&.to_s || '9092'
+        CGI.unescape(url.host).split(',').map do |addr|
+          addr << ':' << port unless addr.match(/:\d+$/)
+          addr
+        end
+      end
+
       def init_producer(**opts)
         @producer = @client.producer(**opts)
       end

data/lib/bps/publisher/kafka_async.rb

@@ -14,18 +14,18 @@ module BPS
       # @param [Hash] opts the options.
       # @option opts [Integer] :max_queue_size (defaults to: 1000)
       #                        the maximum number of messages allowed in the queue.
-      # @option opts [Integer] :delivery_threshold (defaults to: 0)
+      # @option opts [Integer] :delivery_threshold (defaults to: 1000)
       #                        if greater than zero, the number of buffered messages that will automatically
       #                        trigger a delivery.
-      # @option opts [Integer] :delivery_interval (defaults to: 0) if greater than zero, the number of
+      # @option opts [Integer] :delivery_interval (defaults to: 30) if greater than zero, the number of
       #                        seconds between automatic message deliveries.
-      def initialize(broker_addrs, **opts)
+      def initialize(broker_addrs, **opts) # rubocop:disable Lint/UselessMethodDefinition
         super
       end
 
       private
 
-      def init_producer(max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0)
+      def init_producer(max_queue_size: 1000, delivery_threshold: 1000, delivery_interval: 30)
         @client.async_producer(
           max_queue_size: max_queue_size,
           delivery_threshold: delivery_threshold,

data/spec/bps/kafka_spec.rb

@@ -1,38 +1,63 @@
 require 'spec_helper'
 require 'bps/kafka'
 
-def kafka_addrs
-  ENV.fetch('KAFKA_ADDRS', '127.0.0.1:9092').split(',').freeze
-end
+RSpec.describe 'Kafka', kafka: true do
+  context 'resolve addrs' do
+    let(:client) { double('Kafka', producer: nil) }
+    before       { allow(::Kafka).to receive(:new).and_return(client) }
 
-run_spec = \
-  begin
-    ::Kafka.new(kafka_addrs).brokers
-    true
-  rescue StandardError => e
-    warn "WARNING: unable to run #{File.basename __FILE__}: #{e.message}"
-    false
-  end
+    it 'should resolve simple URLs' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        expect(::Kafka).to receive(:new).with(['test.host:9092'], {}).and_return(client)
+      else
+        expect(::Kafka).to receive(:new).with(['test.host:9092']).and_return(client)
+      end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://test.host:9092'))
+    end
 
-helper = proc do
-  def read_messages(topic_name, num_messages)
-    client = ::Kafka.new(kafka_addrs)
-    Enumerator.new do |y|
-      client.each_message(topic: topic_name, start_from_beginning: true) do |msg|
-        y << msg.value
+    it 'should resolve URLs with multiple hosts' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        expect(::Kafka).to receive(:new).with(['foo.host:9092', 'bar.host:9092'], {}).and_return(client)
+      else
+        expect(::Kafka).to receive(:new).with(['foo.host:9092', 'bar.host:9092']).and_return(client)
       end
-    end.take(num_messages)
-  ensure
-    client&.close
-  end
-end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://foo.host,bar.host:9092'))
+    end
 
-RSpec.describe 'Kafka', if: run_spec do
-  context BPS::Publisher::Kafka do
-    it_behaves_like 'publisher', url: "kafka+sync://#{kafka_addrs.join(',')}/", &helper
+    it 'should resolve URLs with multiple hosts/ports' do
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7')
+        expect(::Kafka).to receive(:new).with(['foo.host:9093', 'bar.host:9092'], {}).and_return(client)
+      else
+        expect(::Kafka).to receive(:new).with(['foo.host:9093', 'bar.host:9092']).and_return(client)
+      end
+      BPS::Publisher.resolve(URI.parse('kafka+sync://foo.host%3A9093,bar.host'))
+    end
   end
 
-  context BPS::Publisher::KafkaAsync do
-    it_behaves_like 'publisher', url: "kafka://#{kafka_addrs.join(',')}/", &helper
+  describe 'publishers' do
+    let(:kafka_addrs) { ENV.fetch('KAFKA_ADDRS', '127.0.0.1:9092').split(',') }
+
+    def read_messages(topic_name, num_messages)
+      client = ::Kafka.new(kafka_addrs)
+      Enumerator.new do |y|
+        client.each_message(topic: topic_name, start_from_beginning: true) do |msg|
+          y << msg.value
+        end
+      end.take(num_messages)
+    ensure
+      client&.close
+    end
+
+    context BPS::Publisher::Kafka do
+      let(:publisher_url) { "kafka+sync://#{CGI.escape(kafka_addrs.join(','))}/" }
+
+      it_behaves_like 'publisher'
+    end
+
+    context BPS::Publisher::KafkaAsync do
+      let(:publisher_url) { "kafka://#{CGI.escape(kafka_addrs.join(','))}/" }
+
+      it_behaves_like 'publisher'
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bps-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Black Square Media
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-01 00:00:00.000000000 Z
+date: 2020-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bps
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.1
+        version: 0.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.1
+        version: 0.1.3
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
@@ -69,7 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Kafka adapter for bps