hivent 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e360be776496b49ad0334d22d5bc52d20554829a
-  data.tar.gz: 8a3027f19ecf4bebad1701a0d575f8e56b9a8c4a
+  metadata.gz: bbc8f5896f4204e96e80a1de33bffd52ffe26e0f
+  data.tar.gz: 47590d0d52a5d0ef933bac1aa4f3346ca4b5bfbc
 SHA512:
-  metadata.gz: 76d2ad583fae68f5470124f671c69221d5e9db4ffe76576c73bdf16f75357cf05f3cd6c49fd412450d18f2b2fe1294553caa0bd9ada38b6a59feb433f7e1ba35
-  data.tar.gz: f9ed4579ff32e15235f6c066e10390721e235d88f64971e5f67e1725b30d9b4a58dd3eb364de34c34dff70d719f36a007ade0c64faece3d0263edd9674d6d28b
+  metadata.gz: 42af01f8e3b8152d0b2a90dc963907d28abc10c74d6d5f557519dc66267968bc54c14cb22b39e9f85ea8beaae7c57b4f9d64263e52995448929d88604285273b
+  data.tar.gz: c09144a1cf4542f74ed8c5bfa5d372a9039bcadc8973f0ff89b1d902b92a264765daabe24e3a04831f0d9b4e82b12c4f9997c234364951b585d91f518d1f5e79

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/hivent.gemspec

@@ -22,13 +22,16 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport", "~> 5.0"
   spec.add_dependency "retryable", "~> 2.0"
   spec.add_dependency "redis", "~> 3.3"
-  spec.add_dependency "event_emitter", "~> 0.2"
+  spec.add_dependency "emittr", "~> 0.1"
 
   spec.add_development_dependency "bundler", "~> 1.12"
   spec.add_development_dependency "rspec", "~> 3.5"
   spec.add_development_dependency "rspec-its", "~> 1.2"
+  spec.add_development_dependency "rspec-eventually", "~> 0.2"
   spec.add_development_dependency "pry-byebug", "~> 3.4"
   spec.add_development_dependency "simplecov", "~> 0.12"
   spec.add_development_dependency "codeclimate-test-reporter", "~> 0.6"
   spec.add_development_dependency "rubocop", "~> 0.43"
+  spec.add_development_dependency "gem-release", "~> 0.7"
+  spec.add_development_dependency "rake", "~> 11.3"
 end

data/lib/hivent.rb

@@ -3,7 +3,7 @@ require "active_support"
 require "active_support/core_ext"
 require "retryable"
 require "json"
-require "event_emitter"
+require "emittr"
 
 require "hivent/config"
 

data/lib/hivent/emitter.rb

@@ -3,7 +3,7 @@ module Hivent
 
   class Emitter
 
-    include EventEmitter
+    include Emittr::Events
     attr_accessor :events
 
     WILDCARD = :all
@@ -18,6 +18,10 @@ module Hivent
       end
     end
 
+    def emit(name, *data)
+      super(name.to_sym, *data)
+    end
+
     private
 
     def emittable_event_names(payload)

data/lib/hivent/redis/consumer.rb

@@ -8,6 +8,7 @@ module Hivent
       include Hivent::Redis::Extensions
 
       LUA_CONSUMER = File.expand_path("../lua/consumer.lua", __FILE__)
+      LUA_HEARTBEAT = File.expand_path("../lua/heartbeat.lua", __FILE__)
       # In milliseconds
       SLEEP_TIME   = 200
       CONSUMER_TTL = 1000
@@ -21,22 +22,23 @@ module Hivent
       end
 
       def run!
+        start_heartbeat!
         consume while !@stop
       end
 
       def stop!
         @stop = true
+        stop_heartbeat!
       end
 
       def queues
-        script(LUA_CONSUMER, @service_name, @name, CONSUMER_TTL)
+        script(LUA_CONSUMER, @service_name, @name, CONSUMER_TTL) || []
       end
 
       def consume
         to_process = items
 
         to_process.each do |(queue, item)|
-          @redis.rpop(queue)
           payload = nil
           begin
             payload = JSON.parse(item).with_indifferent_access
@@ -48,6 +50,8 @@ module Hivent
             @redis.lpush(dead_letter_queue_name(queue), item)
 
             @life_cycle_event_handler.event_processing_failed(e, payload, item, dead_letter_queue_name(queue))
+          ensure
+            @redis.rpop(queue)
           end
         end
 
@@ -56,6 +60,26 @@ module Hivent
 
       private
 
+      def start_heartbeat!
+        stop_heartbeat!
+
+        @heartbeat = Thread.new do
+          loop do
+            heartbeat!
+
+            Kernel.sleep(SLEEP_TIME.to_f / 1000)
+          end
+        end
+      end
+
+      def stop_heartbeat!
+        @heartbeat.exit if @heartbeat.present?
+      end
+
+      def heartbeat!
+        script(LUA_HEARTBEAT, @service_name, @name, CONSUMER_TTL)
+      end
+
       def items
         queues
           .map    { |queue| [queue, @redis.lindex(queue, -1)] }

data/lib/hivent/redis/lua/consumer.lua

@@ -50,25 +50,6 @@ local function table_eq(table1, table2)
  return recurse(table1, table2)
 end
 
-local function keepalive(service, consumer)
-  redis.call("SET", service .. ":" .. consumer .. ":alive", "true", "PX", CONSUMER_TTL)
-  redis.call("SADD", service .. ":consumers", consumer)
-end
-
-local function cleanup(service)
-  local consumer_index_key = service .. ":consumers"
-  local consumers = redis.call("SMEMBERS", consumer_index_key)
-
-  for _, consumer in ipairs(consumers) do
-    local consumer_status_key = service .. ":" .. consumer .. ":alive"
-    local alive = redis.call("GET", consumer_status_key)
-
-    if not alive then
-      redis.call("SREM", consumer_index_key, consumer)
-    end
-  end
-end
-
 local function distribute(consumers, partition_count)
   local distribution   = {}
   local consumer_count = table.getn(consumers)
@@ -163,17 +144,4 @@ local function rebalance(service_name, consumer_name)
   end
 end
 
-local function heartbeat(service_name, consumer_name)
-  -- keep consumer alive
-  keepalive(service_name, consumer_name)
-
-  -- clean up dead consumers
-  cleanup(service_name)
-
-  -- rebalance
-  local new_config = rebalance(service_name, consumer_name)
-
-  return new_config
-end
-
-return heartbeat(service_name, consumer_name)
+return rebalance(service_name, consumer_name)

data/lib/hivent/redis/lua/heartbeat.lua

@@ -0,0 +1,34 @@
+local service_name  = ARGV[1]
+local consumer_name = ARGV[2]
+local CONSUMER_TTL  = ARGV[3]
+
+local function keepalive(service, consumer)
+  redis.call("SET", service .. ":" .. consumer .. ":alive", "true", "PX", CONSUMER_TTL)
+  redis.call("SADD", service .. ":consumers", consumer)
+end
+
+local function cleanup(service)
+  local consumer_index_key = service .. ":consumers"
+  local consumers = redis.call("SMEMBERS", consumer_index_key)
+
+  for _, consumer in ipairs(consumers) do
+    local consumer_status_key = service .. ":" .. consumer .. ":alive"
+    local alive = redis.call("GET", consumer_status_key)
+
+    if not alive then
+      redis.call("SREM", consumer_index_key, consumer)
+    end
+  end
+end
+
+local function heartbeat(service_name, consumer_name)
+  -- keep consumer alive
+  keepalive(service_name, consumer_name)
+
+  -- clean up dead consumers
+  cleanup(service_name)
+
+  return true
+end
+
+return heartbeat(service_name, consumer_name)

data/lib/hivent/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Hivent
 
-  VERSION = File.read(File.expand_path('../../../.version', __FILE__)).strip.freeze
+  VERSION = "1.0.5".freeze
 
 end

data/spec/hivent/abstract_signal_spec.rb

@@ -83,7 +83,7 @@ describe Hivent::AbstractSignal do
 
   describe "#receive" do
     after :each do
-      Hivent.emitter.remove_listener name
+      Hivent.emitter.off name
       Hivent.emitter.events.clear
     end
 

data/spec/hivent/redis/consumer_spec.rb

@@ -1,4 +1,3 @@
-# frozen_string_literal: true
 require "spec_helper"
 
 describe Hivent::Redis::Consumer do
@@ -7,12 +6,19 @@ describe Hivent::Redis::Consumer do
   let(:redis)            { Redis.new(url: REDIS_URL) }
   let(:service_name)     { "a_service" }
   let(:consumer_name)    { "a_consumer" }
-  let(:life_cycle_event_handler) { double("Hivent::LifeCycleEventHandler").as_null_object }
+  let(:life_cycle_event_handler) { double("Hivent::Redis::LifeCycleEventHandler").as_null_object }
+
+  before :each do
+    stub_const("#{described_class}::CONSUMER_TTL", 1000)
+  end
 
   after :each do
+    Hivent.emitter.off
     redis.flushall
 
-    Hivent.emitter.__events.clear
+    Thread.list.each do |thread|
+      thread.exit unless thread == Thread.current
+    end
   end
 
   describe "#queues" do
@@ -20,8 +26,8 @@ describe Hivent::Redis::Consumer do
       # 1. Marks every consumer as "alive"
       # 2. Resets every consumer
       # 3. Distributes partitions evenly
-      3.times do
-        consumers.each(&:queues)
+      consumers.map do |consumer|
+        Thread.new { consumer.run! }
       end
     end
 
@@ -34,8 +40,13 @@ describe Hivent::Redis::Consumer do
 
       let(:partition_count) { 2 }
 
+      before :each do
+        Thread.new { consumer.run! }
+        sleep 0.1
+      end
+
       it "returns all available partitions" do
-        expect(subject.length).to eq(partition_count)
+        expect { subject.length }.to eventually eq(partition_count)
       end
     end
 
@@ -48,89 +59,78 @@ describe Hivent::Redis::Consumer do
         before :each do
           # Hearbeat from first consumer,
           # marking it as "alive"
-          consumer1.queues
+          Thread.new { consumer1.run! }
+          sleep 0.1
         end
 
         it "assigns all available partitions to the living consumer" do
-          distribution = [consumer1.queues, consumer2.queues]
-
-          expect(distribution.map(&:length)).to eq([2, 0])
+          expect { [consumer1.queues, consumer2.queues].map(&:length) }.to eventually eq([2, 0])
         end
 
         describe "balancing" do
-          it "resets the first consumer for rebalancing" do
-            # Marks consumer 1 as alive, assigning all partitions
-            consumer1.queues
-            # Marks consumer 2 as alive, assigning 0 partitions to
-            # start rebalancing
-            consumer2.queues
-
-            # Assigns 0 partitions to finish resetting
-            expect(consumer1.queues.length).to eq(0)
-          end
-
           it "assigns half the partitions after reset" do
+            threads = []
             # Fully resets
-            consumer1.queues
-            consumer2.queues
-            consumer1.queues
+            threads << Thread.new { consumer1.run! }
+            sleep 0.1
+            threads << Thread.new { consumer2.run! }
+            sleep 0.1
 
             # Distributes partitions across consumers
-            expect(consumer2.queues.length).to eq(1)
+            expect { consumer2.queues.length }.to eventually eq(1)
           end
 
           it "rebalances partitions across both consumers" do
-            consumer1.queues
-            consumer2.queues
-            consumer1.queues
-            consumer2.queues
+            threads = []
+
+            threads << Thread.new { consumer1.run! }
+            sleep 0.1
+            threads << Thread.new { consumer2.run! }
+            sleep 0.1
+
+            Thread.new { consumer1.stop! }
+            Thread.new { consumer2.stop! }
+            sleep 0.1
+
+            threads << Thread.new { consumer1.run! }
+            sleep 0.1
+            threads << Thread.new { consumer2.run! }
+            sleep 0.1
 
             # Distributes partitions across consumers
-            expect(consumer1.queues.length).to eq(1)
+            expect { consumer1.queues.length }.to eventually eq(1)
           end
 
           context "when one of the consumers dies" do
             before :each do
               stub_const("#{described_class}::CONSUMER_TTL", 50)
 
-              balance([consumer1, consumer2])
-              count = 0
+              threads = balance([consumer1, consumer2])
+              Thread.new { consumer1.stop! }
+              threads.first.exit
 
-              while count <= 2
-                consumer2.queues
-                count += 1
-
-                sleep described_class::CONSUMER_TTL.to_f / 1000
-              end
+              sleep described_class::CONSUMER_TTL.to_f / 1000
             end
 
             it "assigns those consumer's partitions to another consumer" do
-              expect(consumer2.queues.length).to eq(2)
+              expect { consumer2.queues.length }.to eventually eq(2)
             end
           end
         end
       end
 
       context "when both consumers are alive" do
-        subject do
-          [consumer1.queues, consumer2.queues]
-        end
-
         before :each do
           balance([consumer1, consumer2])
         end
 
         it "returns all available partitions" do
-          expect(subject.map(&:length)).to eq([1, 1])
+          expect { [consumer1.queues, consumer2.queues].map(&:length) }.to eventually eq([1, 1])
         end
       end
     end
 
     context "with more consumers than partitions" do
-      subject do
-        [consumer1.queues, consumer2.queues]
-      end
-
       let(:consumer1)       { described_class.new(redis, service_name, "#{consumer_name}1", life_cycle_event_handler) }
       let(:consumer2)       { described_class.new(redis, service_name, "#{consumer_name}2", life_cycle_event_handler) }
       let(:partition_count) { 1 }
@@ -140,7 +140,7 @@ describe Hivent::Redis::Consumer do
       end
 
       it "returns all available partitions" do
-        expect(subject.map(&:length)).to eq([1, 0])
+        expect { [consumer1.queues, consumer2.queues].map(&:length) }.to eventually eq([1, 0])
       end
     end
 
@@ -158,7 +158,7 @@ describe Hivent::Redis::Consumer do
       end
 
       it "returns all available partitions" do
-        expect(subject.map(&:length)).to eq([2, 1])
+        expect { [consumer1.queues, consumer2.queues].map(&:length) }.to eventually eq([2, 1])
       end
     end
 
@@ -184,12 +184,20 @@ describe Hivent::Redis::Consumer do
     let(:producer) { Hivent::Redis::Producer.new(redis) }
 
     before :each do
+      Thread.new { consumer.run! }
+      sleep 0.1
+
       redis.set("#{service_name}:partition_count", partition_count)
       redis.sadd(event[:meta][:name], service_name)
 
       producer.write(event[:meta][:name], event.to_json, 0)
     end
 
+    after :each do
+      Thread.new { consumer.stop! }
+      sleep 0.1
+    end
+
     context "when there are items ready to be consumed" do
 
       it "emits the item with indifferent access" do
@@ -300,24 +308,35 @@ describe Hivent::Redis::Consumer do
   end
 
   describe "#run!" do
-    subject { Thread.new { consumer.run! } }
-
     let(:partition_count) { 2 }
 
     before :each do
       redis.set("#{service_name}:partition_count", partition_count)
 
       allow(consumer).to receive(:consume)
+
+      stub_const("#{described_class}::CONSUMER_TTL", 10000)
     end
 
-    it "processes items" do
-      thread = subject
+    it "starts its heartbeat" do
+      thread = Thread.new { consumer.run! }
 
-      sleep 0.1
+      sleep 1
+
+      is_alive = redis.get("#{service_name}:#{consumer_name}:alive")
 
       thread.kill
 
+      expect(is_alive).to be
+    end
+
+    it "processes items" do
+      thread = Thread.new { consumer.run! }
+      sleep 0.2
+
       expect(consumer).to have_received(:consume).at_least(:once)
+
+      thread.kill
     end
 
   end
@@ -328,6 +347,22 @@ describe Hivent::Redis::Consumer do
 
     before :each do
       redis.set("#{service_name}:partition_count", partition_count)
+      stub_const("#{described_class}::CONSUMER_TTL", 10)
+    end
+
+    it "stops its heartbeat" do
+      thread = Thread.new do
+        consumer.run!
+      end
+
+      sleep 0.1
+
+      consumer.stop!
+      thread.kill
+
+      sleep 0.2
+
+      expect { redis.get("#{service_name}:#{consumer_name}:alive") }.to eventually be_nil
     end
 
     it "stops processing" do
@@ -338,6 +373,7 @@ describe Hivent::Redis::Consumer do
       sleep 0.1
 
       consumer.stop!
+      thread.kill
 
       # nil is returned if timeout expires
       expect(thread.join(2)).to eq(thread)

data/spec/hivent_spec.rb

@@ -83,7 +83,7 @@ describe Hivent do
     let(:increase)  { 5 }
 
     after :each do
-      Hivent.emitter.remove_listener "my_signal:1"
+      Hivent.emitter.off "my_signal:1"
     end
 
     it "consumes events" do

data/spec/spec_helper.rb

@@ -2,6 +2,7 @@
 require 'simplecov'
 
 require 'rspec/its'
+require 'rspec/eventually'
 require 'pry'
 
 require 'hivent'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hivent
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors:
 - Bruno Abrantes
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-10 00:00:00.000000000 Z
+date: 2016-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -53,19 +53,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.3'
 - !ruby/object:Gem::Dependency
-  name: event_emitter
+  name: emittr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: rspec-eventually
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.43'
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.3'
 description: ''
 email:
 - bruno@brunoabrantes.com
@@ -179,10 +221,10 @@ files:
 - ".ruby-version"
 - ".simplecov.template"
 - ".travis.yml"
-- ".version"
 - Gemfile
 - LICENSE
 - README.md
+- Rakefile
 - bin/hivent
 - hivent.gemspec
 - lib/hivent.rb
@@ -197,6 +239,7 @@ files:
 - lib/hivent/redis/consumer.rb
 - lib/hivent/redis/extensions.rb
 - lib/hivent/redis/lua/consumer.lua
+- lib/hivent/redis/lua/heartbeat.lua
 - lib/hivent/redis/lua/producer.lua
 - lib/hivent/redis/producer.rb
 - lib/hivent/redis/redis.rb

data/.version

@@ -1 +0,0 @@
-1.0.4