phobos 1.8.2.pre.beta1 → 1.8.2.pre.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6610d5e383b53c4d2f926b4ab10ae33c260215fe2de0520132fa38e6b51089d8
-  data.tar.gz: '0791b9c0fc4d4b10f878607bddc1374c359bbe627386375ce48ea3ff6996329d'
+  metadata.gz: 1263c2db9d40f0099ff81e41fd27c93c29e18a1d5b7b82db37f22c141db11a67
+  data.tar.gz: 3185ecc3c82100c1667d7aa70e0c7338fd5077ec77aff07676457d637c6acbf1
 SHA512:
-  metadata.gz: dc15fe9776e8aff478e3263a9d8d16b9d693baff6259ae62bdbf1e229b7f8c8bfb345fdf7b9603583ca5405642e1424461c2523ef4bdbcd9b85bd80140a03e6d
-  data.tar.gz: 782ca0a9838cf38a6ae37c92b33d3ce2f4692438a837aeeefabbea629dd6d5b6c595a6cdc7932e804882d29854784583d356574f8adfaa25613fe093e42cf10c
+  metadata.gz: e3010972a7e2de780a046f10c188ec3b65d75c470a84e4f7a64861940a88e95c04dcb99c46cbe08f9ea10ed2ad055fffb13b2ea858891b603f3d5527cd89ab60
+  data.tar.gz: '09778609c9c57778ea845f676024c76a7fce13e3942ba3e4f4beca2571cb88ddb4311e39831ff866a1256d18612f179f05415298b249932c0d97a35fb0c069e1'

data/.rspec

@@ -1,2 +1,3 @@
 --format documentation
 --color
+--backtrace

data/CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## UNRELEASED
+
+## [1.8.2-beta2] - 2019-06-21
+- Added the `persistent_connections` setting and the corresponding
+  `sync_producer_shutdown` method to enable reusing the connection
+  for regular (sync) producers.
+
 ## [1.8.2-beta1] - 2019-03-13
 
 - Added `BatchHandler` to consume messages in batches on the business

data/README.md

@@ -21,6 +21,7 @@ With Phobos by your side, all this becomes smooth sailing.
 ## Table of Contents
 
 1. [Installation](#installation)
+  1. [Upgrade Notes](#upgrade-notes)
 1. [Usage](#usage)
   1. [Standalone apps](#usage-standalone-apps)
   1. [Consuming messages from Kafka](#usage-consuming-messages-from-kafka)
@@ -52,6 +53,10 @@ Or install it yourself as:
 $ gem install phobos
 ```
 
+### <a name="upgrade-notes"></a> Upgrade Notes
+
+Version 1.8.2 introduced a new `persistent_connections` setting for regular producers. This reduces the number of connections used to produce messages and you should consider setting it to true. This does require a manual shutdown call -  please see [Producers with persistent connections](#persistent-connection).
+
 ## <a name="usage"></a> Usage
 
 Phobos can be used in two ways: as a standalone application or to support Kafka features in your existing project - including Rails apps. It provides a CLI tool to run it.
@@ -291,7 +296,7 @@ MyProducer
 
 There are two flavors of producers: __regular__ producers and __async__ producers.
 
-Regular producers will deliver the messages synchronously and disconnect, it doesn't matter if you use `publish` or `publish_list` after the messages get delivered the producer will disconnect.
+Regular producers will deliver the messages synchronously and disconnect, it doesn't matter if you use `publish` or `publish_list`; by default, after the messages get delivered the producer will disconnect.
 
 Async producers will accept your messages without blocking, use the methods `async_publish` and `async_publish_list` to use async producers.
 
@@ -310,7 +315,7 @@ class MyHandler
 end
 ```
 
-#### Note about configuring producers
+#### <a name="producer-config"></a> Note about configuring producers
 
 Since the handler life cycle is managed by the Listener, it will make sure the producer is properly closed before it stops. When calling the producer outside a handler remember, you need to shutdown them manually before you close the application. Use the class method `async_producer_shutdown` to safely shutdown the producer.
 
@@ -328,6 +333,18 @@ MyProducer
   .close
 ```
 
+### <a name="persistent-connection"></a> Note about producers with persistent connections
+
+By default, regular producers will automatically disconnect after every `publish` call. You can change this behavior (which reduces connection overhead, TLS etc - which increases speed significantly) by setting the `persistent_connections` config in `phobos.yml`. When set, regular producers behave identically to async producers and will also need to be shutdown manually using the `sync_producer_shutdown` method.
+
+Since regular producers with persistent connections have open connections, you need to manually disconnect from Kafka when ending your producers' life cycle:
+
+```ruby
+MyProducer
+  .producer
+  .sync_producer_shutdown
+```
+
 ### <a name="usage-as-library"></a> Phobos as a library in an existing project
 
 When running as a standalone service, Phobos sets up a `Listener` and `Executor` for you. When you use Phobos as a library in your own project, you need to set these components up yourself.

data/config/phobos.yml.example

@@ -60,6 +60,11 @@ producer:
   # if greater than zero, the number of seconds between automatic message
   # deliveries. Only used for async_producer
   delivery_interval: 0
+  # Set this to true to keep the producer connection between publish calls.
+  # This can speed up subsequent messages by around 30%, but it does mean
+  # that you need to manually call sync_producer_shutdown before exiting,
+  # similar to async_producer_shutdown.
+  persistent_connections: false
 
 consumer:
   # number of seconds after which, if a client hasn't contacted the Kafka cluster,

data/lib/phobos.rb

@@ -31,6 +31,23 @@ require 'phobos/executor'
 
 Thread.abort_on_exception = true
 
+Logging.init :debug, :info, :warn, :error, :fatal
+
+# Monkey patch to fix this issue: https://github.com/zendesk/ruby-kafka/pull/732
+module Logging
+  # :nodoc:
+  class Logger
+    # :nodoc:
+    def formatter=(*args); end
+
+    # :nodoc:
+    def push_tags(*args); end
+
+    # :nodoc:
+    def pop_tags(*args); end
+  end
+end
+
 module Phobos
   class << self
     attr_reader :config, :logger

data/lib/phobos/producer.rb

@@ -52,6 +52,7 @@ module Phobos
       class PublicAPI
         NAMESPACE = :phobos_producer_store
         ASYNC_PRODUCER_PARAMS = [:max_queue_size, :delivery_threshold, :delivery_interval].freeze
+        INTERNAL_PRODUCER_PARAMS = [:persistent_connections].freeze
 
         # This method configures the kafka client used with publish operations
         # performed by the host class
@@ -67,18 +68,35 @@ module Phobos
           producer_store[:kafka_client]
         end
 
+        def create_sync_producer
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
+          sync_producer = client.producer(regular_configs)
+          if Phobos.config.producer_hash[:persistent_connections]
+            producer_store[:sync_producer] = sync_producer
+          end
+          sync_producer
+        end
+
+        def sync_producer
+          producer_store[:sync_producer]
+        end
+
+        def sync_producer_shutdown
+          sync_producer&.shutdown
+          producer_store[:sync_producer] = nil
+        end
+
         def publish(topic, payload, key = nil, partition_key = nil)
           publish_list([{ topic: topic, payload: payload, key: key,
                           partition_key: partition_key }])
         end
 
         def publish_list(messages)
-          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
-          producer = client.producer(regular_configs)
+          producer = sync_producer || create_sync_producer
           produce_messages(producer, messages)
           producer.deliver_messages
         ensure
-          producer&.shutdown
+          producer&.shutdown unless Phobos.config.producer_hash[:persistent_connections]
         end
 
         def create_async_producer
@@ -109,11 +127,14 @@ module Phobos
         end
 
         def regular_configs
-          Phobos.config.producer_hash.reject { |k, _| ASYNC_PRODUCER_PARAMS.include?(k) }
+          Phobos.config.producer_hash
+                .reject { |k, _| ASYNC_PRODUCER_PARAMS.include?(k) }
+                .reject { |k, _| INTERNAL_PRODUCER_PARAMS.include?(k) }
         end
 
         def async_configs
           Phobos.config.producer_hash
+                .reject { |k, _| INTERNAL_PRODUCER_PARAMS.include?(k) }
         end
 
         private

data/lib/phobos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Phobos
-  VERSION = '1.8.2-beta1'
+  VERSION = '1.8.2-beta2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos
 version: !ruby/object:Gem::Version
-  version: 1.8.2.pre.beta1
+  version: 1.8.2.pre.beta2
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -15,7 +15,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-13 00:00:00.000000000 Z
+date: 2019-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler