delivery_boy 0.2.7 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbf511c98ca6b6d56175d0624aa499f73e431139ca93de2591b8a490af633bc2
-  data.tar.gz: dc7359238540968814b9b13e79fe7f95f80b9f0b441d7e1b2d14575900ea2c48
+  metadata.gz: a12782da215edcc91544e184fd5804b2c21fb4a4be180f69addd0e2f7787884e
+  data.tar.gz: 6de9279ba3441524aceef6091a4feaf57d67a8623499779877b39ed9a11edad9
 SHA512:
-  metadata.gz: 812bf9fc1020bc846ebfbc4853847c6d7cf28649d30e1d5d242e636dd2ce50a87dff8347653afb517b0108d45576358a729d10763054ee93e09084375f5f8462
-  data.tar.gz: d6b30c5fd5ddd38287f8c3fef45cdcc64fe31f0e7f5b92977b1fb8dd338bad2176f487cbff3a547edf2ae1a7a493bfec2f26a2cf8ae3a647d07c9d1e410879d8
+  metadata.gz: 6787bea888f6a2db4bb0d66eaad3c2234c9b6e4482c81ec69e43c27960b4c4a191ec553c0a24c87a46c5067811e2b9eb85265b34fc9457692465dfa8d902c5af
+  data.tar.gz: e10d3020dbaedfe5608fab51e6e1d343341919aab23b905766cdc386e2b844d5177d464fba93d69b79cc6d3704a5ad703ca97464f5754a363a9d362fecec2559

data/.circleci/config.yml

@@ -0,0 +1,33 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.5.1-node
+        environment:
+          LOG_LEVEL: DEBUG
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    steps:
+      - checkout
+      - run: bundle install --path vendor/bundle
+      - run: bundle exec rspec

data/.gitignore

@@ -9,4 +9,4 @@
 /tmp/
 
 # rspec failure tracking
-.rspec_status
+.rspec_status

data/CHANGELOG

@@ -2,6 +2,31 @@
 
 ## Unreleased
 
+## v1.1.0
+
+* Support for `ssl_client_cert_key_password` in the configuration (#52)
+* Add `DeliveryBoy.buffer_size` to return the number of messages in the buffer
+* Add `DeliveryBoy::Fake#clear_buffer` and `DeliveryBoy::Fake#buffer_size` to
+support the public API when using the test helper.
+* Support for `sasl_oauth_token_provider` in the configuration. (#55)
+
+## v1.0.1
+
+* Require ruby-kafka v1.0 or higher.
+
+## v1.0.0
+
+* Add `DeliveryBoy.clear_buffer` method.
+* Support exactly once delivery and transactional messaging (#50)
+* Check that Rails::Railtie is defined (#48)
+
+## v0.2.8
+
+* Support `log_level` config option.
+* Support for ssl_verify_hostname in the configuration (#44)
+* Upgrade dependency on KingKonf.
+* Allow configuring `sasl_over_ssl`.
+
 ## v0.2.7
 
 * Support for ssl_ca_certs_from_system #18

data/README.md

@@ -59,7 +59,34 @@ end
 
 In addition to improving response time, delivering messages asynchronously also protects your application against Kafka availability issues -- if messages cannot be delivered, they'll be buffered for later and retried automatically.
 
-Both `deliver` and `deliver_async` take the following options:
+A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later.
+
+```ruby
+ # app/controllers/comments_controller.rb
+ class CommentsController < ApplicationController
+   def create
+     @comment = Comment.create!(params)
+
+     event = {
+       name: "comment_created",
+       data: {
+         comment_id: @comment.id
+         user_id: current_user.id
+       }
+     }
+
+     # This will queue the two messages in the internal buffer.
+     DeliveryBoy.produce(comment.to_json, topic: "comments")
+     DeliveryBoy.produce(event.to_json, topic: "activity")
+
+     # This will deliver all messages in the buffer to Kafka.
+     # This call is blocking.
+     DeliveryBoy.deliver_messages
+   end
+ end
+```
+
+The methods `deliver`, `deliver_async` and `produce` take the following options:
 
 * `topic` – the Kafka topic that should be written to (required).
 * `key` – the key that should be set on the Kafka message (optional).
@@ -103,6 +130,10 @@ A list of Kafka brokers that should be used to initialize the client. Defaults t
 
 This is how the client will identify itself to the Kafka brokers. Default is `delivery_boy`.
 
+##### `log_level`
+
+The log level for the logger.
+
 #### Message delivery
 
 ##### `delivery_interval`
@@ -189,11 +220,15 @@ A PEM encoded client cert to use with an SSL connection. Must be used in combina
 
 A PEM encoded client cert key to use with an SSL connection. Must be used in combination with `ssl_client_cert`.
 
+##### `ssl_client_cert_key_password`
+
+The password required to read the ssl_client_cert_key. Must be used in combination with ssl_client_cert_key.
+
 #### SASL Authentication and authorization
 
 See [ruby-kafka](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more information.
 
-Use either `sasl_gssapi_*` _or_ `sasl_plain_*`, not both.
+Use it through `GSSAPI`, `PLAIN` _or_ `OAUTHBEARER`.
 
 ##### `sasl_gssapi_principal`
 
@@ -215,6 +250,25 @@ The username used to authenticate.
 
 The password used to authenticate.
 
+##### `sasl_oauth_token_provider`
+
+A instance of a class which implements the `token` method.
+As described in [ruby-kafka](https://github.com/zendesk/ruby-kafka/tree/c3e90bc355fad1e27b9af1048966ff08d3d5735b#oauthbearer)
+
+```ruby
+class TokenProvider
+  def token
+    "oauth-token"
+  end
+end
+
+DeliveryBoy.configure do |config|
+  config.sasl_oauth_token_provider = TokenProvider.new
+  config.ssl_ca_certs_from_system = true
+end
+```
+
+
 ### Testing
 
 DeliveryBoy provides a test mode out of the box. When this mode is enabled, messages will be stored in memory rather than being sent to Kafka. If you use RSpec, enabling test mode is as easy as adding this to your spec helper:
@@ -231,17 +285,17 @@ describe PostsController do
   describe "#show" do
     it "emits an event to Kafka" do
       post = Post.create!(body: "hello")
-      
+
       get :show, id: post.id
-      
+
       # Use this API to extract all messages written to a Kafka topic.
       messages = DeliveryBoy.testing.messages_for("post_views")
-      
+
       expect(messages.count).to eq 1
-      
+
       # In addition to #value, you can also pull out #key and #partition_key.
       event = JSON.parse(messages.first.value)
-      
+
       expect(event["post_id"]).to eq post.id
     end
   end

data/delivery_boy.gemspec

@@ -20,8 +20,8 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "ruby-kafka", "~> 0.5"
-  spec.add_runtime_dependency "king_konf", "~> 0.2"
+  spec.add_runtime_dependency "ruby-kafka", "~> 1.0"
+  spec.add_runtime_dependency "king_konf", "~> 1.0"
 
   spec.add_development_dependency "bundler", "~> 1.15"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/delivery_boy.rb

@@ -4,7 +4,8 @@ require "delivery_boy/version"
 require "delivery_boy/instance"
 require "delivery_boy/fake"
 require "delivery_boy/config"
-require "delivery_boy/railtie" if defined?(Rails)
+require "delivery_boy/config_error"
+require "delivery_boy/railtie" if defined?(Rails::Railtie)
 
 module DeliveryBoy
   class << self
@@ -47,6 +48,49 @@ module DeliveryBoy
       instance.deliver_async!(value, topic: topic, **options)
     end
 
+    # Like {.produce!}, but handles +Kafka::BufferOverflow+ errors
+    # by logging them and just going on with normal business.
+    #
+    # @return [nil]
+    def produce(value, topic:, **options)
+      produce!(value, topic: topic, **options)
+    rescue Kafka::BufferOverflow
+      logger.error "Message for `#{topic}` dropped due to buffer overflow"
+    end
+
+    # Appends the given message to the producer buffer but does not send it until {.deliver_messages} is called.
+    #
+    # @param value [String] the message value.
+    # @param topic [String] the topic that the message should be written to.
+    # @param key [String, nil] the message key.
+    # @param partition [Integer, nil] the topic partition that the message should
+    #   be written to.
+    # @param partition_key [String, nil] a key used to deterministically assign
+    #   a partition to the message.
+    # @return [nil]
+    # @raise [Kafka::BufferOverflow] if the producer's buffer is full.
+    def produce!(value, topic:, **options)
+      instance.produce(value, topic: topic, **options)
+    end
+
+    # Delivers the items currently in the producer buffer.
+    #
+    # @return [nil]
+    # @raise [Kafka::DeliveryFailed] if delivery failed for some reason.
+    def deliver_messages
+      instance.deliver_messages
+    end
+
+    # Clear any buffered messages generated by {.produce} or {.produce!} methods.
+    def clear_buffer
+      instance.clear_buffer
+    end
+
+    # Return the number of messages in the buffer
+    def buffer_size
+      instance.buffer_size
+    end
+
     # Shut down DeliveryBoy.
     #
     # Automatically called when the process exits.
@@ -60,7 +104,11 @@ module DeliveryBoy
     #
     # @return [Logger]
     def logger
-      @logger ||= Logger.new($stdout)
+      @logger ||= Logger.new($stdout).tap do |logger|
+        if config.log_level
+          logger.level = Object.const_get("Logger::#{config.log_level.upcase}")
+        end
+      end
     end
 
     attr_writer :logger

data/lib/delivery_boy/config.rb

@@ -7,6 +7,7 @@ module DeliveryBoy
     # Basic
     list :brokers, items: :string, sep: ",", default: ["localhost:9092"]
     string :client_id, default: "delivery_boy"
+    string :log_level, default: nil
 
     # Buffering
     integer :max_buffer_bytesize, default: 10_000_000
@@ -24,6 +25,9 @@ module DeliveryBoy
     integer :max_retries, default: 2
     integer :required_acks, default: -1
     integer :retry_backoff, default: 1
+    boolean :idempotent, default: false
+    boolean :transactional, default: false
+    integer :transactional_timeout, default: 60
 
     # Compression
     integer :compression_threshold, default: 1
@@ -34,7 +38,9 @@ module DeliveryBoy
     string :ssl_ca_cert_file_path
     string :ssl_client_cert, default: nil
     string :ssl_client_cert_key, default: nil
+    string :ssl_client_cert_key_password, default: nil
     boolean :ssl_ca_certs_from_system, default: false
+    boolean :ssl_verify_hostname, default: true
 
     # SASL authentication
     string :sasl_gssapi_principal
@@ -45,6 +51,10 @@ module DeliveryBoy
     string :sasl_scram_username
     string :sasl_scram_password
     string :sasl_scram_mechanism
+    boolean :sasl_over_ssl, default: true
+
+    # SASL OAUTHBEARER
+    attr_accessor :sasl_oauth_token_provider
 
     # Datadog monitoring
     boolean :datadog_enabled

data/lib/delivery_boy/fake.rb

@@ -2,7 +2,7 @@ module DeliveryBoy
 
   # A fake implementation that is useful for testing.
   class Fake
-    FakeMessage = Struct.new(:value, :topic, :key, :offset, :partition, :partition_key, :create_time) do
+    FakeMessage = Struct.new(:value, :topic, :key, :headers, :offset, :partition, :partition_key, :create_time) do
       def bytesize
         key.to_s.bytesize + value.to_s.bytesize
       end
@@ -10,31 +10,74 @@ module DeliveryBoy
 
     def initialize
       @messages = Hash.new {|h, k| h[k] = [] }
+      @buffer = Hash.new {|h, k| h[k] = [] }
+      @delivery_lock = Mutex.new
     end
 
-    def deliver(value, topic:, key: nil, partition: nil, partition_key: nil, create_time: Time.now)
-      offset = @messages[topic].count
-      message = FakeMessage.new(value, topic, key, offset, partition, partition_key, create_time)
+    def deliver(value, topic:, key: nil, headers: {}, partition: nil, partition_key: nil, create_time: Time.now)
+      @delivery_lock.synchronize do
+        offset = @messages[topic].count
+        message = FakeMessage.new(value, topic, key, headers, offset, partition, partition_key, create_time)
 
-      @messages[topic] << message
+        @messages[topic] << message
+      end
 
       nil
     end
 
     alias deliver_async! deliver
 
+    def produce(value, topic:, key: nil, headers: {}, partition: nil, partition_key: nil, create_time: Time.now)
+      @delivery_lock.synchronize do
+        offset = @buffer[topic].count
+        message = FakeMessage.new(value, topic, key, headers, offset, partition, partition_key, create_time)
+
+        @buffer[topic] << message
+      end
+
+      nil
+    end
+
+    def deliver_messages
+      @delivery_lock.synchronize do
+        @buffer.each do |topic, messages|
+          @messages[topic].push(*messages)
+        end
+        @buffer.clear
+      end
+    end
+
     def shutdown
       clear
     end
 
+    def clear_buffer
+      @delivery_lock.synchronize do
+        @buffer.clear
+      end
+    end
+
+    def buffer_size
+      @delivery_lock.synchronize do
+        @buffer.values.flatten.size
+      end
+    end
+
     # Clear all messages stored in memory.
     def clear
-      @messages.clear
+      @delivery_lock.synchronize do
+        @messages.clear
+        @buffer.clear
+      end
     end
 
     # Return all messages written to the specified topic.
     def messages_for(topic)
-      @messages[topic]
+      @delivery_lock.synchronize do
+        # Return a clone so that the list of messages can be traversed
+        # without worrying about a concurrent modification
+        @messages[topic].clone
+      end
     end
   end
 end

data/lib/delivery_boy/instance.rb

@@ -6,6 +6,7 @@ module DeliveryBoy
     def initialize(config, logger)
       @config = config
       @logger = logger
+      @async_producer = nil
     end
 
     def deliver(value, topic:, **options)
@@ -13,7 +14,7 @@ module DeliveryBoy
       sync_producer.deliver_messages
     rescue
       # Make sure to clear any buffered messages if there's an error.
-      sync_producer.clear_buffer
+      clear_buffer
 
       raise
     end
@@ -27,6 +28,22 @@ module DeliveryBoy
       async_producer.shutdown if async_producer?
     end
 
+    def produce(value, topic:, **options)
+      sync_producer.produce(value, topic: topic, **options)
+    end
+
+    def deliver_messages
+      sync_producer.deliver_messages
+    end
+
+    def clear_buffer
+      sync_producer.clear_buffer
+    end
+
+    def buffer_size
+      sync_producer.buffer_size
+    end
+
     private
 
     attr_reader :config, :logger
@@ -67,7 +84,9 @@ module DeliveryBoy
         ssl_ca_cert_file_path: config.ssl_ca_cert_file_path,
         ssl_client_cert: config.ssl_client_cert,
         ssl_client_cert_key: config.ssl_client_cert_key,
+        ssl_client_cert_key_password: config.ssl_client_cert_key_password,
         ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
+        ssl_verify_hostname: config.ssl_verify_hostname,
         sasl_gssapi_principal: config.sasl_gssapi_principal,
         sasl_gssapi_keytab: config.sasl_gssapi_keytab,
         sasl_plain_authzid: config.sasl_plain_authzid,
@@ -76,6 +95,8 @@ module DeliveryBoy
         sasl_scram_username: config.sasl_scram_username,
         sasl_scram_password: config.sasl_scram_password,
         sasl_scram_mechanism: config.sasl_scram_mechanism,
+        sasl_over_ssl: config.sasl_over_ssl,
+        sasl_oauth_token_provider: config.sasl_oauth_token_provider
       )
     end
 
@@ -90,6 +111,9 @@ module DeliveryBoy
         max_buffer_bytesize: config.max_buffer_bytesize,
         compression_codec: (config.compression_codec.to_sym if config.compression_codec),
         compression_threshold: config.compression_threshold,
+        idempotent: config.idempotent,
+        transactional: config.transactional,
+        transactional_timeout: config.transactional_timeout,
       }
     end
   end

data/lib/delivery_boy/version.rb

@@ -1,3 +1,3 @@
 module DeliveryBoy
-  VERSION = "0.2.7"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: delivery_boy
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 1.1.0
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-19 00:00:00.000000000 Z
+date: 2021-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: king_konf
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +87,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -97,7 +98,6 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- circle.yml
 - delivery_boy.gemspec
 - examples/async.rb
 - examples/sync.rb
@@ -130,8 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A simple way to produce messages to Kafka from Ruby applications

data/circle.yml

@@ -1,16 +0,0 @@
-machine:
-  pre:
-    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
-  services:
-    - docker
-  ruby:
-    version: 2.4.1
-
-dependencies:
-  pre:
-    - docker -v
-    - docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=localhost --env ADVERTISED_PORT=9092 -d spotify/kafka
-
-test:
-  override:
-    - bundle exec rspec