waterdrop 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ceca692ac4b8b16fe33f636d44f3cda6ebb2be7
-  data.tar.gz: 16385fd05baf2dd90deff2b6c7c6a6b4ca585bdd
+  metadata.gz: 787b506aa11e17d65df6abb198bbd86b750b95a8
+  data.tar.gz: 248d7a49d44adb5379940b64e7d5fd27b3821557
 SHA512:
-  metadata.gz: 48c546123bcd03a06cda425d00e78c5fa235a59d19a87e20142962b6f5d41e4e671a63a5b882425481ab59bae20a684ef52ca7ca229828ae3c878972bc1de0ad
-  data.tar.gz: 1f25b37359947f74dc1cd85dd8b34728ac629b9a6cc3bce311656fb0aefae69a47c6d4f898cc9b1ef353b8e8a85e2be1d50ef0e17cf820553e82fe1eb21e2cf7
+  metadata.gz: b161ff34041a2471d19fea75d770c3f4e2eb1e5bdccd6ddca10863cc08444846c31aef795a3b69c779e239f1579e823326857ea78ab74d728570af69d0a0c397
+  data.tar.gz: 1713e4c8d7556c7d66d1ec76a848fcfbd8eb5b38ed7230f1b4518606f30232314a69915bf5bafc3a1bc8c038f267518bc23642e2d60293c810281e567d352845

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # WaterDrop changelog
 
+## 0.3.0
+- Driver change from Poseidon (not maintained) to Ruby-Kafka
+
 ## 0.2.0
 - Version dump - this WaterDrop version no longer relies on Aspector to work
 - #17 - Logger for Aspector - WaterDrop no longer depends on Aspector

data/Gemfile.lock

@@ -1,58 +1,49 @@
 PATH
   remote: .
   specs:
-    waterdrop (0.2.1)
+    waterdrop (0.3.0)
       bundler
       connection_pool
       null-logger
-      poseidon
       rake
+      ruby-kafka
 
 GEM
   remote: https://rubygems.org/
   specs:
-    abstract_type (0.0.7)
-    activemodel (4.2.5)
-      activesupport (= 4.2.5)
+    activemodel (4.2.6)
+      activesupport (= 4.2.6)
       builder (~> 3.1)
-    activesupport (4.2.5)
+    activesupport (4.2.6)
       i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
       thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    adamantium (0.2.0)
-      ice_nine (~> 0.11.0)
-      memoizable (~> 0.4.0)
+    addressable (2.4.0)
     ast (2.2.0)
-    astrolabe (1.3.1)
-      parser (~> 2.2)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
-    brakeman (3.1.4)
+    brakeman (3.2.1)
       erubis (~> 2.6)
-      fastercsv (~> 1.5)
       haml (>= 3.0, < 5.0)
       highline (>= 1.6.20, < 2.0)
-      multi_json (~> 1.2)
-      ruby2ruby (>= 2.1.1, < 2.3.0)
-      ruby_parser (~> 3.7.0)
+      ruby2ruby (~> 2.3.0)
+      ruby_parser (~> 3.8.1)
       safe_yaml (>= 1.0)
       sass (~> 3.0)
       slim (>= 1.3.6, < 4.0)
       terminal-table (~> 1.4)
-    bson (4.0.0)
+    bson (4.0.4)
     builder (3.2.2)
-    codeclimate-engine-rb (0.1.0)
+    codeclimate-engine-rb (0.3.1)
       virtus (~> 1.0)
-    coderay (1.1.0)
+    coderay (1.1.1)
     coercible (1.0.0)
       descendants_tracker (~> 0.0.1)
-    concord (0.1.5)
-      adamantium (~> 0.2.0)
-      equalizer (~> 0.0.9)
+    colorize (0.7.7)
     connection_pool (2.2.0)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
@@ -60,10 +51,10 @@ GEM
     docile (1.1.5)
     equalizer (0.0.11)
     erubis (2.7.0)
-    faker (1.6.1)
+    faker (1.6.3)
       i18n (~> 0.5)
-    fastercsv (1.5.5)
-    flay (2.6.1)
+    flay (2.7.0)
+      erubis (~> 2.7.0)
       ruby_parser (~> 3.0)
       sexp_processor (~> 4.0)
     flog (4.3.2)
@@ -77,28 +68,27 @@ GEM
       sysexits (~> 1.1)
     highline (1.7.8)
     i18n (0.7.0)
-    ice_nine (0.11.1)
+    ice_nine (0.11.2)
     json (1.8.3)
-    memoizable (0.4.2)
-      thread_safe (~> 0.3, >= 0.3.1)
+    launchy (2.4.3)
+      addressable (~> 2.3)
     method_source (0.8.2)
-    minitest (5.8.3)
-    mongo (2.2.1)
+    minitest (5.8.4)
+    mongo (2.2.4)
       bson (~> 4.0)
-    mongoid (5.0.2)
+    mongoid (5.1.1)
       activemodel (~> 4.0)
       mongo (~> 2.1)
-      origin (~> 2.1)
+      origin (~> 2.2)
       tzinfo (>= 0.3.37)
     mongoid-rspec (3.0.0)
       mongoid (~> 5.0)
       rake
       rspec (~> 3.3)
-    multi_json (1.11.2)
-    null-logger (0.1.0)
-    origin (2.1.1)
-    parser (2.2.3.0)
-      ast (>= 1.1, < 3.0)
+    null-logger (0.1.1)
+    origin (2.2.0)
+    parser (2.3.0.6)
+      ast (~> 2.2)
     polishgeeks-dev-tools (1.2.1)
       brakeman
       faker
@@ -112,64 +102,62 @@ GEM
       simplecov
       timecop
       yard
-    poseidon (0.0.5)
     powerpack (0.1.1)
-    private_attr (1.1.0)
-    procto (0.0.2)
     pry (0.10.3)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    rainbow (2.0.0)
-    rake (10.4.2)
-    reek (3.7.1)
-      codeclimate-engine-rb (~> 0.1.0)
-      parser (~> 2.2, >= 2.2.2.5)
-      private_attr (~> 1.1)
+    rainbow (2.1.0)
+    rake (11.1.1)
+    reek (3.11)
+      codeclimate-engine-rb (~> 0.3.1)
+      parser (~> 2.3, >= 2.3.0.6)
       rainbow (~> 2.0)
-      unparser (~> 0.2.2)
     rspec (3.4.0)
       rspec-core (~> 3.4.0)
       rspec-expectations (~> 3.4.0)
       rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.1)
+    rspec-core (3.4.4)
       rspec-support (~> 3.4.0)
     rspec-expectations (3.4.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.4.0)
-    rspec-mocks (3.4.0)
+    rspec-mocks (3.4.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.4.0)
     rspec-support (3.4.1)
-    rubocop (0.35.1)
-      astrolabe (~> 1.3)
-      parser (>= 2.2.3.0, < 3.0)
+    rubocop (0.38.0)
+      parser (>= 2.3.0.6, < 3.0)
       powerpack (~> 0.1)
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
-      tins (<= 1.6.0)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-kafka (0.3.2)
     ruby-progressbar (1.7.5)
-    ruby2ruby (2.2.0)
+    ruby2ruby (2.3.0)
       ruby_parser (~> 3.1)
       sexp_processor (~> 4.0)
-    ruby_parser (3.7.2)
+    ruby_parser (3.8.1)
       sexp_processor (~> 4.1)
-    rubycritic (2.4.0)
-      flay (= 2.6.1)
+    rubycritic (2.8.0)
+      colorize
+      flay (= 2.7.0)
       flog (= 4.3.2)
-      parser (>= 2.2.0, < 3.0)
-      reek (= 3.7.1)
+      launchy (= 2.4.3)
+      parser (~> 2.3)
+      reek (= 3.11)
+      ruby_parser (~> 3.8)
       virtus (~> 1.0)
     safe_yaml (1.0.4)
-    sass (3.4.20)
-    sexp_processor (4.6.0)
+    sass (3.4.21)
+    sexp_processor (4.7.0)
     shoulda (3.5.0)
       shoulda-context (~> 1.0, >= 1.0.1)
       shoulda-matchers (>= 1.4.1, < 3.0)
     shoulda-context (1.2.1)
     shoulda-matchers (2.8.0)
       activesupport (>= 3.0.0)
-    simplecov (0.11.1)
+    simplecov (0.11.2)
       docile (~> 1.1.0)
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
@@ -182,19 +170,11 @@ GEM
     temple (0.7.6)
     terminal-table (1.5.2)
     thread_safe (0.3.5)
-    tilt (2.0.1)
+    tilt (2.0.2)
     timecop (0.8.0)
-    tins (1.6.0)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    unparser (0.2.4)
-      abstract_type (~> 0.0.7)
-      adamantium (~> 0.2.0)
-      concord (~> 0.1.5)
-      diff-lcs (~> 1.2.5)
-      equalizer (~> 0.0.9)
-      parser (~> 2.2.2)
-      procto (~> 0.0.2)
+    unicode-display_width (1.0.2)
     virtus (1.0.5)
       axiom-types (~> 0.1)
       coercible (~> 1.0)

data/README.md

@@ -3,6 +3,7 @@
 [![Build Status](https://travis-ci.org/karafka/waterdrop.png)](https://travis-ci.org/karafka/waterdrop)
 [![Code Climate](https://codeclimate.com/github/karafka/waterdrop/badges/gpa.svg)](https://codeclimate.com/github/karafka/waterdrop)
 [![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](http://badge.fury.io/rb/waterdrop)
+[![Join the chat at https://gitter.im/karafka/karafka](https://badges.gitter.im/karafka/karafka.svg)](https://gitter.im/karafka/karafka?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Gem used to send messages to Kafka in an easy way.
 

data/lib/water_drop/config.rb

@@ -18,7 +18,7 @@ module WaterDrop
       kafka_hosts
       send_messages
       raise_on_failure
-    )
+    ).freeze
 
     OPTIONS.each do |attr_name|
       attr_accessor attr_name

data/lib/water_drop/message.rb

@@ -21,11 +21,8 @@ module WaterDrop
     def send!
       return true unless ::WaterDrop.config.send_messages?
 
-      Pool.with do |producer|
-        producer.send_messages([
-          Poseidon::MessageToSend.new(topic, message)
-        ])
-      end
+      Pool.with { |producer| producer.send_message(self) }
+
       ::WaterDrop.logger.info("Message #{message} was sent to topic '#{topic}'")
     rescue StandardError => e
       # Even if we dont reraise this exception, it should log that it happened

data/lib/water_drop/pool.rb

@@ -1,5 +1,5 @@
 module WaterDrop
-  # Raw poseidon connection pool for WaterDrop messages delivery
+  # Raw Kafka producers connection pool for WaterDrop messages delivery
   module Pool
     extend SingleForwardable
     # Delegate directly to pool

data/lib/water_drop/producer_proxy.rb

@@ -1,31 +1,20 @@
 module WaterDrop
   # Proxy object for a producer (sender) objects that are inside pool
   # We use it to provide additional timeout monitoring layer
-  # There seem to be some issues with Poseidon and usage of sockets that
-  # are old and not used - that's why we just reinitialize connection if
-  # the connection layer is not being used for too long
+  # There used to be an issue with Poseidon (previous engine for this lib)
+  # usage of sockets that are old and not used - that's why we just
+  # reinitialize connection if the connection layer is not being used for too long
+  # We keep this logic to avoid problems just in case. If those problems won't occure
+  # with Ruby-Kafka, we will drop it
   class ProducerProxy
     # How long should be object considered alive if nothing is being
     # send using it. After that time, we will recreate the connection
     LIFE_TIME = 5 * 60 #  5 minute
 
     # If sending fails - how many times we should try with a new connection
-    # @note It works in a similar way to Poseidon internal max_send_retries option, but it will
-    #   create a new connection after failure (Poseidon tries to use the same one)
     MAX_SEND_RETRIES = 1
 
-    # All default poseidon parameters that we want to use
-    POSEIDON_PARAMS = {
-      metadata_refresh_interval_ms: 5 * 60 * 1000, # 5 minutes
-      # @see https://kafka.apache.org/08/configuration.html
-      # Security level for producer
-      required_acks: -1,
-      # @see https://issues.apache.org/jira/browse/KAFKA-1494
-      retry_backoff_ms: 1000,
-      max_send_retries: 1
-    }
-
-    # @return [WaterDrop::ProducerProxy] proxy object to Poseidon::Producer
+    # @return [WaterDrop::ProducerProxy] proxy object to Kafka::Producer
     # @note To ignore @last_usage nil case - we just assume that it is being
     #   first used when we create it
     def initialize
@@ -33,17 +22,16 @@ module WaterDrop
       @attempts = 0
     end
 
-    # Sends messages to Kafka
-    # @param messages [Array<Poseidon::MessageToSend>] array with messages that we want to send
-    # @return [Boolean] were the messages send
-    # @note Even if you send one message - it still needs to be in an array
+    # Sends message to Kafka
+    # @param message [WaterDrop::Message] message that we want to send
     # @note If something goes wrong it will assume that producer is corrupted and will try to
     #   create a new one
     # @example Send 1 message
-    #   ProducerProxy.new.send_messages([Poseidon::MessageToSend.new(topic, message)])
-    def send_messages(messages)
+    #   ProducerProxy.new.send_message(WaterDrop::Message.new(topic, message))
+    def send_message(message)
       touch
-      producer.send_messages(messages)
+      producer.produce(message.message, topic: message.topic)
+      producer.deliver_messages
     rescue StandardError => e
       reload!
 
@@ -61,20 +49,10 @@ module WaterDrop
       @last_usage = Time.now
     end
 
-    # @return [Poseidon::Producer] producer instance to which we can forward method requests
+    # @return [Kafka::Producer] producer instance to which we can forward method requests
     def producer
       reload! if dead?
-      # Metadata refresh interval needs to be in miliseconds
-      @producer ||= Poseidon::Producer.new(
-        ::WaterDrop.config.kafka_hosts,
-        producer_id,
-        POSEIDON_PARAMS
-      )
-    end
-
-    # @return [String] random unique id for producer
-    def producer_id
-      object_id.to_s + Time.now.to_f.to_s
+      @producer ||= Kafka.new(seed_brokers: ::WaterDrop.config.kafka_hosts).producer
     end
 
     # @return [Boolean] true if we cannot use producer anymore because it was not used for a
@@ -85,6 +63,7 @@ module WaterDrop
 
     # Resets a producer so a new one will be created once requested
     def reload!
+      @producer.shutdown if @producer
       @producer = nil
     end
   end

data/lib/water_drop/version.rb

@@ -2,5 +2,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '0.2.1'.freeze
+  VERSION = '0.3.0'.freeze
 end

data/lib/waterdrop.rb

@@ -6,7 +6,7 @@
   logger
   pathname
   json
-  poseidon
+  kafka
   forwardable
   connection_pool
   null_logger

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'bundler', '>= 0'
   spec.add_dependency 'rake', '>= 0'
-  spec.add_dependency 'poseidon', '>= 0'
+  spec.add_dependency 'ruby-kafka', '>= 0'
   spec.add_dependency 'connection_pool', '>= 0'
   spec.add_dependency 'null-logger'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -40,7 +40,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: poseidon
+  name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="