delivery_boy 0.2.8.beta1 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8b6f4609e705c72433240e90bd6c7bdda86f5ca99fdd9faf9d44f229d9f1e7e
-  data.tar.gz: 5db195d68c9eae6afd34a87984ec18e91e99f6976362a83f840abcc4d5de8117
+  metadata.gz: 7d1d10c8e6cf173eeca342252bf3e47327f08200fd6e34207e803d6ef490ead7
+  data.tar.gz: 4e7d841926323cc46aefec6cdf93812796ae2a454a562e4ad0c6d02eb79b229e
 SHA512:
-  metadata.gz: bb243655a65186433f0dde3b36a9c9c7fb6240f68e238afb361b57c5458732e7152745fb8026f67331986cecc7ceb9f1bb2319c4d76ba778141dc088ae412608
-  data.tar.gz: 19eeaf3237e612814226602c671b16f374d8c881da1a8e1095f15ea0785e0c9fff3453d238133f1bfb60d5897e9760b84a8cbe6759c68853f9e2dbd58065811a
+  metadata.gz: 6388bf8cadefd13a20ceabb7e887171e0f6c8dc42cb5f31a5370da7578bb5e5913192a20161d0370af43cc3a88798f74417886657f887e636fec2586dddfd27a
+  data.tar.gz: e1cbea934e040fcea51b9730e0340376aef0f59ede9c8d82199bba3b0adc60c309ef87a86ade3b7a6bb1d9f7136403dfb6d64408693974f4832f85a04ce24500

data/.circleci/config.yml

@@ -0,0 +1,33 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.5.1-node
+        environment:
+          LOG_LEVEL: DEBUG
+      - image: wurstmeister/zookeeper
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9092
+          KAFKA_PORT: 9092
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9093
+          KAFKA_PORT: 9093
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+      - image: wurstmeister/kafka:2.11-2.0.0
+        environment:
+          KAFKA_ADVERTISED_HOST_NAME: localhost
+          KAFKA_ADVERTISED_PORT: 9094
+          KAFKA_PORT: 9094
+          KAFKA_ZOOKEEPER_CONNECT: localhost:2181
+          KAFKA_DELETE_TOPIC_ENABLE: true
+    steps:
+      - checkout
+      - run: bundle install --path vendor/bundle
+      - run: bundle exec rspec

data/.gitignore

@@ -9,4 +9,4 @@
 /tmp/
 
 # rspec failure tracking
-.rspec_status
+.rspec_status

data/CHANGELOG

@@ -4,7 +4,10 @@
 
 ## v0.2.8
 
+* Support `log_level` config option.
+* Support for ssl_verify_hostname in the configuration (#44)
 * Upgrade dependency on KingKonf.
+* Allow configuring `sasl_over_ssl`.
 
 ## v0.2.7
 

data/README.md

@@ -59,7 +59,34 @@ end
 
 In addition to improving response time, delivering messages asynchronously also protects your application against Kafka availability issues -- if messages cannot be delivered, they'll be buffered for later and retried automatically.
 
-Both `deliver` and `deliver_async` take the following options:
+A third method is to produce messages first (without delivering the messages to Kafka yet), and deliver them synchronously later.
+
+```ruby
+ # app/controllers/comments_controller.rb
+ class CommentsController < ApplicationController
+   def create
+     @comment = Comment.create!(params)
+     
+     event = {
+       name: "comment_created",
+       data: {
+         comment_id: @comment.id
+         user_id: current_user.id
+       }
+     }
+     
+     # This will queue the two messages in the internal buffer.
+     DeliveryBoy.produce(comment.to_json, topic: "comments")
+     DeliveryBoy.produce(event.to_json, topic: "activity")
+     
+     # This will deliver all messages in the buffer to Kafka.
+     # This call is blocking. 
+     DeliveryBoy.deliver_messages
+   end
+ end
+```
+
+The methods `deliver`, `deliver_async` and `produce` take the following options:
 
 * `topic` – the Kafka topic that should be written to (required).
 * `key` – the key that should be set on the Kafka message (optional).
@@ -103,6 +130,10 @@ A list of Kafka brokers that should be used to initialize the client. Defaults t
 
 This is how the client will identify itself to the Kafka brokers. Default is `delivery_boy`.
 
+##### `log_level`
+
+The log level for the logger.
+
 #### Message delivery
 
 ##### `delivery_interval`
@@ -231,17 +262,17 @@ describe PostsController do
   describe "#show" do
     it "emits an event to Kafka" do
       post = Post.create!(body: "hello")
-      
+
       get :show, id: post.id
-      
+
       # Use this API to extract all messages written to a Kafka topic.
       messages = DeliveryBoy.testing.messages_for("post_views")
-      
+
       expect(messages.count).to eq 1
-      
+
       # In addition to #value, you can also pull out #key and #partition_key.
       event = JSON.parse(messages.first.value)
-      
+
       expect(event["post_id"]).to eq post.id
     end
   end

data/delivery_boy.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "ruby-kafka", "~> 0.5"
+  spec.add_runtime_dependency "ruby-kafka", "~> 0.7.8"
   spec.add_runtime_dependency "king_konf", "~> 0.3"
 
   spec.add_development_dependency "bundler", "~> 1.15"

data/lib/delivery_boy.rb

@@ -47,6 +47,39 @@ module DeliveryBoy
       instance.deliver_async!(value, topic: topic, **options)
     end
 
+    # Like {.produce!}, but handles +Kafka::BufferOverflow+ errors
+    # by logging them and just going on with normal business.
+    #
+    # @return [nil]
+    def produce(value, topic:, **options)
+      produce!(value, topic: topic, **options)
+    rescue Kafka::BufferOverflow
+      logger.error "Message for `#{topic}` dropped due to buffer overflow"
+    end
+
+    # Appends the given message to the producer buffer but does not send it until {.deliver_messages} is called.
+    #
+    # @param value [String] the message value.
+    # @param topic [String] the topic that the message should be written to.
+    # @param key [String, nil] the message key.
+    # @param partition [Integer, nil] the topic partition that the message should
+    #   be written to.
+    # @param partition_key [String, nil] a key used to deterministically assign
+    #   a partition to the message.
+    # @return [nil]
+    # @raise [Kafka::BufferOverflow] if the producer's buffer is full.
+    def produce!(value, topic:, **options)
+      instance.produce(value, topic: topic, **options)
+    end
+
+    # Delivers the items currently in the producer buffer.
+    #
+    # @return [nil]
+    # @raise [Kafka::DeliveryFailed] if delivery failed for some reason.
+    def deliver_messages
+      instance.deliver_messages
+    end
+
     # Shut down DeliveryBoy.
     #
     # Automatically called when the process exits.
@@ -60,7 +93,11 @@ module DeliveryBoy
     #
     # @return [Logger]
     def logger
-      @logger ||= Logger.new($stdout)
+      @logger ||= Logger.new($stdout).tap do |logger|
+        if config.log_level
+          logger.level = Object.const_get("Logger::#{config.log_level.upcase}")
+        end
+      end
     end
 
     attr_writer :logger

data/lib/delivery_boy/config.rb

@@ -7,6 +7,7 @@ module DeliveryBoy
     # Basic
     list :brokers, items: :string, sep: ",", default: ["localhost:9092"]
     string :client_id, default: "delivery_boy"
+    string :log_level, default: nil
 
     # Buffering
     integer :max_buffer_bytesize, default: 10_000_000
@@ -35,6 +36,7 @@ module DeliveryBoy
     string :ssl_client_cert, default: nil
     string :ssl_client_cert_key, default: nil
     boolean :ssl_ca_certs_from_system, default: false
+    boolean :ssl_verify_hostname, default: true
 
     # SASL authentication
     string :sasl_gssapi_principal
@@ -45,6 +47,7 @@ module DeliveryBoy
     string :sasl_scram_username
     string :sasl_scram_password
     string :sasl_scram_mechanism
+    boolean :sasl_over_ssl, default: true
 
     # Datadog monitoring
     boolean :datadog_enabled

data/lib/delivery_boy/fake.rb

@@ -10,31 +10,62 @@ module DeliveryBoy
 
     def initialize
       @messages = Hash.new {|h, k| h[k] = [] }
+      @buffer = Hash.new {|h, k| h[k] = [] }
+      @delivery_lock = Mutex.new
     end
 
     def deliver(value, topic:, key: nil, partition: nil, partition_key: nil, create_time: Time.now)
-      offset = @messages[topic].count
-      message = FakeMessage.new(value, topic, key, offset, partition, partition_key, create_time)
+      @delivery_lock.synchronize do
+        offset = @messages[topic].count
+        message = FakeMessage.new(value, topic, key, offset, partition, partition_key, create_time)
 
-      @messages[topic] << message
+        @messages[topic] << message
+      end
 
       nil
     end
 
     alias deliver_async! deliver
 
+    def produce(value, topic:, key: nil, partition: nil, partition_key: nil, create_time: Time.now)
+      @delivery_lock.synchronize do
+        offset = @buffer[topic].count
+        message = FakeMessage.new(value, topic, key, offset, partition, partition_key, create_time)
+
+        @buffer[topic] << message
+      end
+
+      nil
+    end
+
+    def deliver_messages
+      @delivery_lock.synchronize do
+        @buffer.each do |topic, messages|
+          @messages[topic].push(*messages)
+        end
+        @buffer.clear
+      end
+    end
+
     def shutdown
       clear
     end
 
     # Clear all messages stored in memory.
     def clear
-      @messages.clear
+      @delivery_lock.synchronize do
+        @messages.clear
+        @buffer.clear
+      end
     end
 
     # Return all messages written to the specified topic.
     def messages_for(topic)
-      @messages[topic]
+      @delivery_lock.synchronize do
+        # Return a clone so that the list of messages can be traversed
+        # without worrying about a concurrent modification
+        @messages[topic].clone
+      end
     end
   end
 end

data/lib/delivery_boy/instance.rb

@@ -6,6 +6,7 @@ module DeliveryBoy
     def initialize(config, logger)
       @config = config
       @logger = logger
+      @async_producer = nil
     end
 
     def deliver(value, topic:, **options)
@@ -27,6 +28,14 @@ module DeliveryBoy
       async_producer.shutdown if async_producer?
     end
 
+    def produce(value, topic:, **options)
+      sync_producer.produce(value, topic: topic, **options)
+    end
+
+    def deliver_messages
+      sync_producer.deliver_messages
+    end
+
     private
 
     attr_reader :config, :logger
@@ -68,6 +77,7 @@ module DeliveryBoy
         ssl_client_cert: config.ssl_client_cert,
         ssl_client_cert_key: config.ssl_client_cert_key,
         ssl_ca_certs_from_system: config.ssl_ca_certs_from_system,
+        ssl_verify_hostname: config.ssl_verify_hostname,
         sasl_gssapi_principal: config.sasl_gssapi_principal,
         sasl_gssapi_keytab: config.sasl_gssapi_keytab,
         sasl_plain_authzid: config.sasl_plain_authzid,
@@ -76,6 +86,7 @@ module DeliveryBoy
         sasl_scram_username: config.sasl_scram_username,
         sasl_scram_password: config.sasl_scram_password,
         sasl_scram_mechanism: config.sasl_scram_mechanism,
+        sasl_over_ssl: config.sasl_over_ssl
       )
     end
 

data/lib/delivery_boy/version.rb

@@ -1,3 +1,3 @@
 module DeliveryBoy
-  VERSION = "0.2.8.beta1"
+  VERSION = "0.2.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: delivery_boy
 version: !ruby/object:Gem::Version
-  version: 0.2.8.beta1
+  version: 0.2.8
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-21 00:00:00.000000000 Z
+date: 2019-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.7.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.7.8
 - !ruby/object:Gem::Dependency
   name: king_konf
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +87,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -97,7 +98,6 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- circle.yml
 - delivery_boy.gemspec
 - examples/async.rb
 - examples/sync.rb
@@ -126,12 +126,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A simple way to produce messages to Kafka from Ruby applications

data/circle.yml

@@ -1,16 +0,0 @@
-machine:
-  pre:
-    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
-  services:
-    - docker
-  ruby:
-    version: 2.4.1
-
-dependencies:
-  pre:
-    - docker -v
-    - docker run -p 2181:2181 -p 9092:9092 --env ADVERTISED_HOST=localhost --env ADVERTISED_PORT=9092 -d spotify/kafka
-
-test:
-  override:
-    - bundle exec rspec