active_publisher 1.1.1.pre0-java → 1.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 67b8fcd037ff45746b81591c39dda1a031a91c73
-  data.tar.gz: 48d2f53ffac8ab519c618bb860d2760812f14a63
+SHA256:
+  metadata.gz: 9feb8c420d646d0eda3073f0f1971f9b616087667229ed178bde560b508b7d19
+  data.tar.gz: 5601e3a0bcd7d8497192193bca84bae8153feef915806de4bcc90db966e579f4
 SHA512:
-  metadata.gz: 86df2154a9225fcc25a02081da27dc5b5c2753beb18d167edde392d394e96add3bfdfe3c93803d765c73f7ddf7b1ac3d99cd1aa936a8ae33a48d79be5dc9a17c
-  data.tar.gz: 655086d301b518f5e6747eef10972529943c02af529e3097a174ec369da3ba4c1514452c3dd3a1332b4cbf00e1d24f59ee454d6f2370cdf1debb26c6c543811d
+  metadata.gz: 29faadeaaf24e76b1cf34b19fa7d8c8e0437c2c1427f9fbc698cb9f05719563cb0b869dc111b8e7a4129203b6b6d21cf2ce586e8b1d8c9333d3630681fc0f0df
+  data.tar.gz: 7cf68c45c074c3880c39a3b5ecbd82f54686a648039b47e24e9f0aee84e8e6a98421a71f5cd451990f8dcdab306287517d0817b5976e7bb6f67110c2f440868b

data/active_publisher.gemspec

@@ -30,7 +30,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'multi_op_queue', '>= 0.2.0'
 
+  spec.add_development_dependency "benchmark-ips"
   spec.add_development_dependency "bundler"
+  spec.add_development_dependency "connection_pool"
+  spec.add_development_dependency "fakeredis"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2"

data/lib/active_publisher/async/in_memory_adapter/async_queue.rb

@@ -71,7 +71,7 @@ module ActivePublisher
 
             # Check to see if we should restart the consumer.
             if !consumer.alive? || consumer_is_lagging
-              consumer.kill
+              consumer.kill rescue nil
               @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
             end
 

data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb

@@ -101,7 +101,7 @@ module ActivePublisher
                 raise unknown_error
               ensure
                 # Always requeue anything that gets stuck.
-                queue.concat(current_messages) unless current_messages.nil?
+                queue.concat(current_messages) if current_messages && !current_messages.empty?
               end
             end
           end

data/lib/active_publisher/async/redis_adapter/consumer.rb

@@ -0,0 +1,42 @@
+require "active_publisher/async/redis_adapter/redis_multi_pop_queue"
+
+module ActivePublisher
+  module Async
+    module RedisAdapter
+      class Consumer
+        SUPERVISOR_INTERVAL = {
+          :execution_interval => 10, # seconds
+          :timeout_interval => 5, # seconds
+        }
+
+        attr_reader :consumer, :queue, :supervisor
+
+        def initialize(redis_pool)
+          @queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
+          create_and_supervise_consumer!
+        end
+
+        def create_and_supervise_consumer!
+          @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+
+          supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
+            # This may also be the place to start additional publishers when we are getting backed up ... ?
+            unless consumer.alive?
+              consumer.kill rescue nil
+              @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+            end
+
+            # Notify the current queue size.
+            ::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
+          end
+
+          supervisor_task.execute
+        end
+
+        def size
+          queue.size
+        end
+      end
+    end
+  end
+end

data/lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb

@@ -0,0 +1,113 @@
+module ActivePublisher
+  module Async
+    module RedisAdapter
+      class RedisMultiPopQueue
+        attr_reader :list_key, :redis_pool
+
+        def initialize(redis_connection_pool, new_list_key)
+          @redis_pool = redis_connection_pool
+          @list_key = new_list_key
+        end
+
+        def <<(message)
+          encoded_message = ::Marshal.dump(message)
+
+          redis_pool.with do |redis|
+            redis.rpush(list_key, encoded_message)
+          end
+        end
+
+        def concat(*messages)
+          messages = messages.flatten
+          messages.compact!
+          return if messages.empty?
+
+          encoded_messages = []
+          messages.each do |message|
+            encoded_messages << ::Marshal.dump(message)
+          end
+
+          redis_pool.with do |redis|
+            redis.rpush(list_key, encoded_messages)
+          end
+        end
+
+        def empty?
+          size <= 0
+        end
+
+        def pop_up_to(num_to_pop, opts = {})
+          case opts
+          when TrueClass, FalseClass
+            non_bock = opts
+          when Hash
+            timeout = opts.fetch(:timeout, nil)
+            non_block = opts.fetch(:non_block, false)
+          end
+
+          queue_size = size
+          if queue_size <= 0
+            if non_block
+              raise ThreadError, "queue empty"
+            else
+              total_waited_time = 0
+
+              loop do
+                total_waited_time += 0.1
+                sleep 0.1
+                queue_size = size
+
+                if queue_size > 0
+                  num_to_pop = [num_to_pop, queue_size].min # make sure we don't pop more than size
+                  return shift(num_to_pop)
+                end
+
+                return if timeout && total_waited_time > timeout
+              end
+            end
+          else
+            shift(num_to_pop)
+          end
+        end
+
+        def shift(number)
+          number = [number, size].min
+          return [] if number <= 0
+
+          messages = []
+          multi_response = []
+          redis_pool.with do |redis|
+            multi_response = redis.multi do
+              redis.lrange(list_key, 0, number - 1)
+              redis.ltrim(list_key, number, -1)
+            end
+          end
+
+          messages = multi_response.first
+          success = multi_response.last
+          return [] if multi_response.size != 2 || success.nil?
+          return [] unless success =~ /ok/i
+
+          messages = [] if messages.nil?
+          messages = [messages] unless messages.respond_to?(:each)
+
+          shifted_messages = []
+          messages.each do |message|
+            next if message.nil?
+
+            shifted_messages << ::Marshal.load(message)
+          end
+
+          shifted_messages
+        end
+
+        def size
+          redis_pool.with do |redis|
+            redis.llen(list_key) || 0
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/active_publisher/async/redis_adapter.rb

@@ -0,0 +1,74 @@
+require "active_publisher"
+require "active_publisher/message"
+require "active_publisher/async/redis_adapter/consumer"
+require "multi_op_queue"
+
+module ActivePublisher
+  module Async
+    module RedisAdapter
+      REDIS_LIST_KEY = "ACTIVE_PUBLISHER_LIST".freeze
+
+      def self.new(*args)
+        ::ActivePublisher::Async::RedisAdapter::Adapter.new(*args)
+      end
+
+      class Adapter
+        SUPERVISOR_INTERVAL = {
+          :execution_interval => 1.5, # seconds
+          :timeout_interval => 1, # seconds
+        }
+        include ::ActivePublisher::Logging
+
+        attr_reader :async_queue, :redis_pool, :queue
+
+        def initialize(new_redis_pool)
+          logger.info "Starting redis publisher adapter"
+          # do something with supervision ?
+          @redis_pool = new_redis_pool
+          @async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
+          @queue = ::MultiOpQueue::Queue.new
+
+          supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
+            queue_size = queue.size
+            number_of_times = [queue_size / 50, 1].max # get the max number of times to flush
+            number_of_times = [number_of_times, 5].min # don't allow it to be more than 5 per run
+            number_of_times.times { flush_queue! }
+          end
+
+          supervisor_task.execute
+        end
+
+        def publish(route, payload, exchange_name, options = {})
+          message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
+          queue << ::Marshal.dump(message)
+          flush_queue! if queue.size >= 20 || options[:flush_queue]
+
+          nil
+        end
+
+        def shutdown!
+          logger.info "Draining async publisher redis adapter before shutdown."
+          flush_queue! until queue.empty?
+          # Sleeping 2.1 seconds because the most common redis `fsync` command in AOF mode is run every 1 second
+          # this will give at least 1 full `fsync` to run before the process dies
+          sleep 2.1
+        end
+
+      private
+
+        def flush_queue!
+          return if queue.empty?
+          encoded_messages = queue.pop_up_to(25, :timeout => 0.001)
+
+          return if encoded_messages.nil?
+          return unless encoded_messages.respond_to?(:each)
+          return unless encoded_messages.size > 0
+
+          redis_pool.with do |redis|
+            redis.rpush(::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY, encoded_messages)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_publisher/configuration.rb

@@ -1,4 +1,5 @@
 require "yaml"
+require "erb"
 
 module ActivePublisher
   class Configuration

data/lib/active_publisher/version.rb

@@ -1,3 +1,3 @@
 module ActivePublisher
-  VERSION = "1.1.1.pre0"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_publisher
 version: !ruby/object:Gem::Version
-  version: 1.1.1.pre0
+  version: 1.2.0
 platform: java
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-01-10 00:00:00.000000000 Z
+date: 2018-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -70,6 +70,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: benchmark-ips
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -84,6 +98,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: connection_pool
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: fakeredis
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -154,6 +196,9 @@ files:
 - lib/active_publisher/async/in_memory_adapter/async_queue.rb
 - lib/active_publisher/async/in_memory_adapter/channel.rb
 - lib/active_publisher/async/in_memory_adapter/consumer_thread.rb
+- lib/active_publisher/async/redis_adapter.rb
+- lib/active_publisher/async/redis_adapter/consumer.rb
+- lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb
 - lib/active_publisher/configuration.rb
 - lib/active_publisher/connection.rb
 - lib/active_publisher/logging.rb
@@ -174,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 2.6.11
 signing_key:
 specification_version: 4
 summary: Aims to make publishing work across MRI and jRuby painless and add some nice