active_publisher 1.1.1 → 1.2.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f3edc99c354f7cf507f117d4ddfbb13974f3abc
-  data.tar.gz: 32663bda945c4519d5767a205a32ea931eee689f
+  metadata.gz: af572797671347ce2affc2ec5c7dff8b677157b9
+  data.tar.gz: 78780a6e31a452f4b61b0980850ca2ebccc26bd2
 SHA512:
-  metadata.gz: 0b08a1749006c13b88febda66c6b10b7fd5c6c6b1b8e4f84ee92a96fc2f7362c6d79c212b1481cc751126e30a228a2e6ef904f5b1128278f25e26a9035fc0331
-  data.tar.gz: 7cd33f6f2960ca931fd8e40ff8a38d373b40479e70a09efe8cbb96bc6438771ca716c17bf88202ee32efbfaa4314a5b8ae36856102487a9976fb0ed340a3d1b6
+  metadata.gz: 556f432ec930501acef50a143cf02e11398e8bb8a38007d9126a0e3a27966fe873cba272e1ecbb8b3c7d231514f2e0b074cb593a6c7466c8681ecc3eb663424e
+  data.tar.gz: d92c6f495bd7c085c6b5995197d2dc9985c8e5e5a373358f75015ee7f59b3b7b66590920d166d9185d11fe9585a0dbad30077f3179a91129ced0ffb9eaf47cec

data/active_publisher.gemspec

@@ -30,7 +30,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'multi_op_queue', '>= 0.2.0'
 
+  spec.add_development_dependency "benchmark-ips"
   spec.add_development_dependency "bundler"
+  spec.add_development_dependency "connection_pool"
+  spec.add_development_dependency "fakeredis"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2"

data/lib/active_publisher/async/in_memory_adapter/async_queue.rb

@@ -71,7 +71,7 @@ module ActivePublisher
 
             # Check to see if we should restart the consumer.
             if !consumer.alive? || consumer_is_lagging
-              consumer.kill
+              consumer.kill rescue nil
               @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
             end
 

data/lib/active_publisher/async/redis_adapter/consumer.rb

@@ -0,0 +1,39 @@
+require "active_publisher/async/redis_adapter/redis_multi_pop_queue"
+
+module ActivePublisher
+  module Async
+    module RedisAdapter
+      class Consumer
+        SUPERVISOR_INTERVAL = {
+          :execution_interval => 10, # seconds
+          :timeout_interval => 5, # seconds
+        }
+
+        attr_reader :consumer, :queue, :supervisor
+
+        def initialize(redis_pool)
+          @queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
+          create_and_supervise_consumer!
+        end
+
+        def create_and_supervise_consumer!
+          @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+
+          supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
+            # This may also be the place to start additional publishers when we are getting backed up ... ?
+            unless consumer.alive?
+              consumer.kill rescue nil
+              @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+            end
+
+            # Notify the current queue size.
+            ::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
+          end
+
+          supervisor_task.execute
+        end
+      end
+
+    end
+  end
+end

data/lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb

@@ -0,0 +1,98 @@
+module ActivePublisher
+  module Async
+    module RedisAdapter
+      class RedisMultiPopQueue
+        attr_reader :list_key, :redis_pool
+
+        def initialize(redis_connection_pool, new_list_key)
+          @redis_pool = redis_connection_pool
+          @list_key = new_list_key
+        end
+
+        def <<(message)
+          encoded_message = ::Marshal.dump(message)
+
+          redis_pool.with do |redis|
+            redis.lpush(list_key, encoded_message)
+          end
+        end
+
+        def concat(*messages)
+          messages = messages.flatten
+          messages.compact!
+
+          encoded_messages = []
+          messages.each do |message|
+            encoded_messages << ::Marshal.dump(message)
+          end
+
+          redis_pool.with do |redis|
+            redis.lpush(list_key, encoded_messages)
+          end
+        end
+
+        def empty?
+          size <= 0
+        end
+
+        def pop_up_to(num_to_pop, opts = {})
+          case opts
+          when TrueClass, FalseClass
+            non_bock = opts
+          when Hash
+            timeout = opts.fetch(:timeout, nil)
+            non_block = opts.fetch(:non_block, false)
+          end
+
+          queue_size = size
+          if queue_size <= 0
+            if non_block
+              raise ThreadError, "queue empty"
+            else
+              total_waited_time = 0
+
+              loop do
+                total_waited_time += 0.1
+                sleep 0.1
+                queue_size = size
+
+                if queue_size > 0
+                  num_to_pop = [num_to_pop, queue_size].min # make sure we don't pop more than size
+                  return shift(num_to_pop)
+                end
+
+                return if timeout && total_waited_time > timeout
+              end
+            end
+          else
+            shift(num_to_pop)
+          end
+        end
+
+        def shift(number)
+          messages = []
+          number = [number, size].min
+          redis_pool.with do |redis|
+            redis.pipelined do
+              number.times do
+                messages << redis.rpop(list_key)
+              end
+            end
+          end
+
+          messages = [] if messages.nil?
+          messages = [messages] unless messages.respond_to?(:each)
+          messages.compact!
+          messages.map { |message| ::Marshal.load(message.value) }
+        end
+
+        def size
+          redis_pool.with do |redis|
+            redis.llen(list_key) || 0
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/active_publisher/async/redis_adapter.rb

@@ -0,0 +1,73 @@
+require "active_publisher/message"
+require "active_publisher/async/redis_adapter/consumer"
+require "multi_op_queue"
+
+module ActivePublisher
+  module Async
+    module RedisAdapter
+      REDIS_LIST_KEY = "ACTIVE_PUBLISHER_LIST".freeze
+
+      def self.new(*args)
+        ::ActivePublisher::Async::RedisAdapter::Adapter.new(*args)
+      end
+
+      class Adapter
+        SUPERVISOR_INTERVAL = {
+          :execution_interval => 1.5, # seconds
+          :timeout_interval => 1, # seconds
+        }
+        include ::ActivePublisher::Logging
+
+        attr_reader :async_queue, :redis_pool, :queue
+
+        def initialize(new_redis_pool)
+          logger.info "Starting redis publisher adapter"
+          # do something with supervision ?
+          @redis_pool = new_redis_pool
+          @async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
+          @queue = ::MultiOpQueue::Queue.new
+
+          supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
+            queue_size = queue.size
+            number_of_times = [queue_size / 50, 1].max # get the max number of times to flush
+            number_of_times = [number_of_times, 5].min # don't allow it to be more than 5 per run
+            number_of_times.times { flush_queue! }
+          end
+
+          supervisor_task.execute
+        end
+
+        def publish(route, payload, exchange_name, options = {})
+          message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
+          queue << ::Marshal.dump(message)
+          flush_queue! if queue.size >= 20 || options[:flush_queue]
+
+          nil
+        end
+
+        def shutdown!
+          logger.info "Draining async publisher redis adapter before shutdown."
+          flush_queue! until queue.empty?
+          # Sleeping 2.1 seconds because the most common redis `fsync` command in AOF mode is run every 1 second
+          # this will give at least 1 full `fsync` to run before the process dies
+          sleep 2.1
+        end
+
+      private
+
+        def flush_queue!
+          return if queue.empty?
+          encoded_messages = queue.pop_up_to(25, :timeout => 0.001)
+
+          return if encoded_messages.nil?
+          return unless encoded_messages.respond_to?(:each)
+          return unless encoded_messages.size > 0
+
+          redis_pool.with do |redis|
+            redis.lpush(::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY, encoded_messages)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_publisher/configuration.rb

@@ -1,4 +1,5 @@
 require "yaml"
+require "erb"
 
 module ActivePublisher
   class Configuration

data/lib/active_publisher/version.rb

@@ -1,3 +1,3 @@
 module ActivePublisher
-  VERSION = "1.1.1"
+  VERSION = "1.2.0.pre"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_publisher
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0.pre
 platform: ruby
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-11 00:00:00.000000000 Z
+date: 2018-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -70,6 +70,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +98,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +196,9 @@ files:
 - lib/active_publisher/async/in_memory_adapter/async_queue.rb
 - lib/active_publisher/async/in_memory_adapter/channel.rb
 - lib/active_publisher/async/in_memory_adapter/consumer_thread.rb
+- lib/active_publisher/async/redis_adapter.rb
+- lib/active_publisher/async/redis_adapter/consumer.rb
+- lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb
 - lib/active_publisher/configuration.rb
 - lib/active_publisher/connection.rb
 - lib/active_publisher/logging.rb
@@ -174,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Aims to make publishing work across MRI and jRuby painless and add some nice