bps-kafka 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5c35677b45d9df97c7ab28ef3ce4e910d2edcd6def70caa235923b48ab9cd057
+  data.tar.gz: 5d4ee5dce8765945dd11a373e4d461e1c5aaf08ee32e77f7e40c64954d934f2e
+SHA512:
+  metadata.gz: be7fcda9c77f2f3bb4d29f49bb29ecce4362f0bd41405e6567b93f9ce4c81019e522d3d7bfecb81d52e67d468bbc8e807c562d35cac8a05f1cf01d03d0a7eb61
+  data.tar.gz: af85befa5ff134a58df21d247b648b72d2c81206ffe1613c92ae8270d43bcb3ba3d12e9adcd31f3832ff0c2966d04e2ae377f93d5a3c8ff348a360e70f0021a5

data/bps-kafka.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |s|
+  s.name        = 'bps-kafka'
+  s.version     = File.read(File.expand_path('../../.version', __dir__)).strip
+  s.platform    = Gem::Platform::RUBY
+
+  s.licenses    = ['Apache-2.0']
+  s.summary     = 'Kafka adapter for bps'
+  s.description = 'https://github.com/bsm/bps'
+
+  s.authors     = ['Black Square Media']
+  s.email       = 'info@blacksquaremedia.com'
+  s.homepage    = 'https://github.com/bsm/bps'
+
+  s.executables   = []
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ['lib']
+  s.required_ruby_version = '>= 2.6.0'
+
+  s.add_dependency 'bps', s.version
+  s.add_dependency 'ruby-kafka', '>= 1.1.0.beta1'
+end

data/lib/bps-kafka.rb

@@ -0,0 +1 @@
+require 'bps/kafka'

data/lib/bps/kafka.rb

@@ -0,0 +1,18 @@
+require 'bps'
+require 'kafka'
+require 'bps/publisher/kafka'
+require 'bps/publisher/kafka_async'
+
+module BPS
+  module Publisher
+    register('kafka+sync') do |url, **opts|
+      addrs = CGI.unescape(url.host).split(',')
+      Kafka.new(addrs, **Kafka.coercer.coerce(opts))
+    end
+
+    register('kafka') do |url, **opts|
+      addrs = CGI.unescape(url.host).split(',')
+      KafkaAsync.new(addrs, **Kafka.coercer.coerce(opts))
+    end
+  end
+end

data/lib/bps/publisher/kafka.rb

@@ -0,0 +1,102 @@
+require 'bps/kafka'
+
+module BPS
+  module Publisher
+    class Kafka < Abstract
+      class Topic
+        def initialize(producer, topic)
+          @producer = producer
+          @topic = topic
+        end
+
+        def publish(message, **opts)
+          @producer.produce(message, **opts, topic: @topic)
+          after_publish
+        end
+
+        def flush
+          @producer.deliver_messages
+        end
+
+        protected
+
+        def after_publish
+          @producer.deliver_messages
+          nil
+        end
+      end
+
+      CLIENT_OPTS = {
+        client_id: :string,
+        connect_timeout: :float,
+        socket_timeout: :float,
+        ssl_ca_cert_file_path: :string,
+        ssl_ca_cert: :string,
+        ssl_client_cert: :string,
+        ssl_client_cert_key: :string,
+        ssl_client_cert_key_password: :string,
+        ssl_client_cert_chain: :string,
+        sasl_gssapi_principal: :string,
+        sasl_gssapi_keytab: :string,
+        sasl_plain_authzid: :string,
+        sasl_plain_username: :string,
+        sasl_plain_password: :string,
+        sasl_scram_username: :string,
+        sasl_scram_password: :string,
+        sasl_scram_mechanism: :string,
+        sasl_over_ssl: :bool,
+        ssl_ca_certs_from_system: :bool,
+        ssl_verify_hostname: :bool,
+      }.freeze
+
+      PRODUCER_OPTS = {
+        # standard
+        retry_backoff: :float,
+        compression_codec: :symbol,
+        compression_threshold: :int,
+        ack_timeout: :float,
+        required_acks: :symbol,
+        max_retries: :int,
+        max_buffer_size: :int,
+        max_buffer_bytesize: :int,
+        idempotent: :bool,
+        transactional: :bool,
+        transactional_id: :string,
+        transactional_timeout: :bool,
+        # async
+        delivery_interval: :float,
+        delivery_threshold: :int,
+        max_queue_size: :int,
+      }.freeze
+
+      # @return [BPS::Coercer] the options coercer.
+      def self.coercer
+        @coercer ||= BPS::Coercer.new(CLIENT_OPTS.merge(PRODUCER_OPTS)).freeze
+      end
+
+      # @param [Array<String>] brokers the seed broker addresses.
+      # @param [Hash] opts the options.
+      # @see https://www.rubydoc.info/gems/ruby-kafka/Kafka/Client#initialize-instance_method
+      def initialize(broker_addrs, **opts)
+        @topics   = {}
+        @client   = ::Kafka.new(broker_addrs, **opts.slice(*CLIENT_OPTS.keys))
+        @producer = init_producer(**opts.slice(*PRODUCER_OPTS.keys))
+      end
+
+      def topic(name)
+        @topics[name] ||= self.class::Topic.new(@producer, name)
+      end
+
+      def close
+        @producer.shutdown
+        @client.close
+      end
+
+      private
+
+      def init_producer(**opts)
+        @producer = @client.producer(**opts)
+      end
+    end
+  end
+end

data/lib/bps/publisher/kafka_async.rb

@@ -0,0 +1,37 @@
+require 'bps/publisher/kafka'
+
+module BPS
+  module Publisher
+    class KafkaAsync < Kafka
+      class Topic < Kafka::Topic
+        private
+
+        def after_publish; end
+      end
+
+      # @see BPS::Kafka::Publisher
+      #
+      # @param [Hash] opts the options.
+      # @option opts [Integer] :max_queue_size (defaults to: 1000)
+      #                        the maximum number of messages allowed in the queue.
+      # @option opts [Integer] :delivery_threshold (defaults to: 0)
+      #                        if greater than zero, the number of buffered messages that will automatically
+      #                        trigger a delivery.
+      # @option opts [Integer] :delivery_interval (defaults to: 0) if greater than zero, the number of
+      #                        seconds between automatic message deliveries.
+      def initialize(broker_addrs, **opts)
+        super
+      end
+
+      private
+
+      def init_producer(max_queue_size: 1000, delivery_threshold: 0, delivery_interval: 0)
+        @client.async_producer(
+          max_queue_size: max_queue_size,
+          delivery_threshold: delivery_threshold,
+          delivery_interval: delivery_interval,
+        )
+      end
+    end
+  end
+end

data/spec/bps/kafka_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+require 'bps/kafka'
+
+def kafka_addrs
+  ENV.fetch('KAFKA_ADDRS', '127.0.0.1:9092').split(',').freeze
+end
+
+run_spec = \
+  begin
+    ::Kafka.new(kafka_addrs).brokers
+    true
+  rescue StandardError => e
+    warn "WARNING: unable to run #{File.basename __FILE__}: #{e.message}"
+    false
+  end
+
+helper = proc do
+  def read_messages(topic_name, num_messages)
+    client = ::Kafka.new(kafka_addrs)
+    Enumerator.new do |y|
+      client.each_message(topic: topic_name, start_from_beginning: true) do |msg|
+        y << msg.value
+      end
+    end.take(num_messages)
+  ensure
+    client&.close
+  end
+end
+
+RSpec.describe 'Kafka', if: run_spec do
+  context BPS::Publisher::Kafka do
+    it_behaves_like 'publisher', url: "kafka+sync://#{kafka_addrs.join(',')}/", &helper
+  end
+
+  context BPS::Publisher::KafkaAsync do
+    it_behaves_like 'publisher', url: "kafka://#{kafka_addrs.join(',')}/", &helper
+  end
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: bps-kafka
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Black Square Media
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bps
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: ruby-kafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0.beta1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0.beta1
+description: https://github.com/bsm/bps
+email: info@blacksquaremedia.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bps-kafka.gemspec
+- lib/bps-kafka.rb
+- lib/bps/kafka.rb
+- lib/bps/publisher/kafka.rb
+- lib/bps/publisher/kafka_async.rb
+- spec/bps/kafka_spec.rb
+homepage: https://github.com/bsm/bps
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key: 
+specification_version: 4
+summary: Kafka adapter for bps
+test_files:
+- spec/bps/kafka_spec.rb