asynchronic 1.6.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d090602f5e7db3b74e67c5057d7aec4761c2007e
-  data.tar.gz: b2e6ce2ff2d236510a71d91d7d5e0042179e4a64
+  metadata.gz: 7a0200ef4e9c9c465ddc70786d71e15bb2fa2f4d
+  data.tar.gz: 0d5320739a21a0cd57f7e65cfe28d6dea571dbfb
 SHA512:
-  metadata.gz: dd1f22974965986b3ce74f2187edf7ad7911bee32562871a7591cea31a93fa22187beaa8b3e1fcf416a09560585460c51557669cc07c067eed6007d31d7959ed
-  data.tar.gz: 1a8db5dddd1501792e2bc00cfe57be14b0ef0c8f6e44d8c29a77c03bb0ce0f80d0cda483f87b84703f474a245e3bed611909e2496e0fc44e444cebca6b1bab82
+  metadata.gz: 2294c1215c7b3abb787c3ec9cd955b254b9b1ec235e6c028d1ff9620b881bc0f0a7ed937b98b14e1ef2537b0be2921e67772ccaec1d9b37616bccc8a11fbe8cf
+  data.tar.gz: 8fa59d9eeafa9a952a9c76c2b91c807d4eaa0fcbc58ee2b1bed1bd637358b42f4c2ef8e1fb1bb864bad86664e45c67fbc44e8bd8e2099dedda6b6e08e95e6cc2

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 
 rvm:
-  - 1.9.3
   - 2.0
   - 2.1
   - 2.2
@@ -9,7 +8,6 @@ rvm:
   - 2.4.0
   - 2.5.0
   - 2.6.0
-  - jruby-1.7.25
   - jruby-9.1.7.0
   - ruby-head
   - jruby-head

data/asynchronic.gemspec

@@ -18,8 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'redis', '~> 3.0'
-  spec.add_dependency 'ost', '~> 0.1'
+  spec.add_dependency 'ost', '~> 1.0'
   spec.add_dependency 'class_config', '~> 0.0'
   spec.add_dependency 'transparent_proxy', '~> 0.0'
   spec.add_dependency 'multi_require', '~> 1.0'
@@ -33,10 +32,4 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov', '~> 0.12'
   spec.add_development_dependency 'coveralls', '~> 0.8'
   spec.add_development_dependency 'pry-nav', '~> 0.2'
-
-  if RUBY_VERSION < '2'
-    spec.add_development_dependency 'term-ansicolor', '~> 1.3.0'
-    spec.add_development_dependency 'tins', '~> 1.6.0'
-    spec.add_development_dependency 'json', '~> 1.8'
-  end
 end

data/lib/asynchronic.rb

@@ -1,11 +1,13 @@
 require 'forwardable'
 require 'securerandom'
-require 'redis'
 require 'ost'
+require 'redic'
 require 'class_config'
 require 'transparent_proxy'
 require 'logger'
 require 'multi_require'
+require 'timeout'
+require 'socket'
 
 MultiRequire.require_relative_pattern 'asynchronic/**/*.rb'
 
@@ -20,6 +22,8 @@ module Asynchronic
   attr_config :retry_timeout, 30
   attr_config :garbage_collector_timeout, 30
   attr_config :redis_data_store_sync_timeout, 0.01
+  attr_config :keep_alive_timeout, 0.1
+  attr_config :connection_name, "HOST=#{Socket.gethostname},PID=#{::Process.pid}"
 
   def self.environment
     Environment.new queue_engine, data_store

data/lib/asynchronic/data_store/lazy_value.rb

@@ -18,7 +18,7 @@ module Asynchronic
       end
 
       def data_store
-        @data_store_class.connect *@data_store_connection_args
+        @data_store_class.connect(*@data_store_connection_args)
       end
 
       def to_value

data/lib/asynchronic/data_store/redis.rb

@@ -8,11 +8,11 @@ module Asynchronic
 
       def initialize(scope, *args)
         @scope = Key[scope]
-        @connection = ::Redis.new(*args)
+        @redis = Redic.new(*args)
       end
 
       def [](key)
-        value = @connection.get @scope[key]
+        value = @redis.call 'GET', @scope[key]
         value ? Marshal.load(value) : nil
       rescue => ex
         Asynchronic.logger.warn('Asynchronic') { ex.message }
@@ -20,33 +20,33 @@ module Asynchronic
       end
 
       def []=(key, value)
-        @connection.set @scope[key], Marshal.dump(value)
+        @redis.call 'SET', @scope[key], Marshal.dump(value)
       end
 
       def delete(key)
-        @connection.del @scope[key]
+        @redis.call 'DEL', @scope[key]
       end
 
       def delete_cascade(key)
-        @connection.del @scope[key]
-        @connection.keys(@scope[key]['*']).each { |k| @connection.del k }
+        @redis.call 'DEL', @scope[key]
+        @redis.call('KEYS', @scope[key]['*']).each { |k| @redis.call 'DEL', k }
       end
 
       def keys
-        @connection.keys(@scope['*']).map { |k| Key[k].remove_first }
+        @redis.call('KEYS', @scope['*']).map { |k| Key[k].remove_first }
       end
 
       def synchronize(key)
-        while @connection.getset(@scope[key][LOCKED], LOCKED) == LOCKED
+        while @redis.call('GETSET', @scope[key][LOCKED], LOCKED) == LOCKED
           sleep Asynchronic.redis_data_store_sync_timeout
         end
         yield
       ensure
-        @connection.del @scope[key][LOCKED]
+        @redis.call 'DEL', @scope[key][LOCKED]
       end
 
       def connection_args
-        [@scope, @connection.client.options]
+        [@scope, @redis.url]
       end
 
       def self.connect(*args)

data/lib/asynchronic/process.rb

@@ -11,7 +11,7 @@ module Asynchronic
       aborted:   :finalized_at
     }
 
-    ATTRIBUTE_NAMES = [:type, :name, :queue, :status, :dependencies, :data, :result, :error] | TIME_TRACKING_MAP.values.uniq
+    ATTRIBUTE_NAMES = [:type, :name, :queue, :status, :dependencies, :data, :result, :error, :connection_name] | TIME_TRACKING_MAP.values.uniq
 
     CANCELED_ERROR_MESSAGE = 'Canceled'
 
@@ -47,6 +47,10 @@ module Asynchronic
       abort! CANCELED_ERROR_MESSAGE
     end
 
+    def dead?
+      (running? && !connected?) || processes.any?(&:dead?)
+    end
+
     def destroy
       data_store.delete_cascade
     end
@@ -75,7 +79,7 @@ module Asynchronic
 
     def processes
       data_store.scoped(:processes).keys.
-        select { |k| k.sections.count == 2 && k.match(/name$/) }.
+        select { |k| k.sections.count == 2 && k.match(/\|name$/) }.
         sort.map { |k| Process.new environment, id[:processes][k.remove_last] }
     end
 
@@ -208,6 +212,8 @@ module Asynchronic
     end
 
     def run
+      self.connection_name = Asynchronic.connection_name
+
       if root.aborted?
         abort!
       else
@@ -215,6 +221,7 @@ module Asynchronic
         self.result = job.call
         waiting!
       end
+    
     rescue Exception => ex
       message = "Failed process #{type} (#{id})\n#{ex.class} #{ex.message}\n#{ex.backtrace.join("\n")}"
       Asynchronic.logger.error('Asynchronic') { message }
@@ -249,5 +256,9 @@ module Asynchronic
       parent.queue if parent
     end
 
+    def connected?
+      connection_name && environment.queue_engine.active_connections.include?(connection_name)
+    end
+
   end
 end

data/lib/asynchronic/queue_engine/in_memory.rb

@@ -33,6 +33,11 @@ module Asynchronic
         true
       end
 
+      def active_connections
+        [Asynchronic.connection_name]
+      end
+
+
       class Queue
 
         extend Forwardable

data/lib/asynchronic/queue_engine/ost.rb

@@ -1,17 +1,14 @@
 module Asynchronic
   module QueueEngine
     class Ost
-      
+
       attr_reader :default_queue
 
       def initialize(options={})
-        ::Ost.connect options[:redis] if options.key?(:redis)
-        @default_queue = options[:default_queue]
+        @redis = Redic.new(*Array(options[:redis]))
+        @default_queue = options.fetch(:default_queue, Asynchronic.default_queue)
         @queues ||= Hash.new { |h,k| h[k] = Queue.new k }
-      end
-
-      def default_queue
-        @default_queue ||= Asynchronic.default_queue
+        @keep_alive_thread = notify_keep_alive
       end
 
       def [](name)
@@ -19,12 +16,12 @@ module Asynchronic
       end
 
       def queues
-        (@queues.values.map(&:key) | redis.keys('ost:*')).map { |q| q.to_s[4..-1].to_sym }
+        (@queues.values.map(&:key) | @redis.call('KEYS', 'ost:*')).map { |q| q.to_s[4..-1].to_sym }
       end
 
       def clear
         @queues.clear
-        redis.keys('ost:*').each { |k| redis.del k }
+        @redis.call('KEYS', 'ost:*').each { |k| @redis.call('DEL', k) }
       end
 
       def listener
@@ -35,21 +32,32 @@ module Asynchronic
         true
       end
 
+      def active_connections
+        @redis.call('CLIENT', 'LIST').split("\n").map do |connection_info|
+          connection_info.split(' ').detect { |a| a.match(/name=/) }[5..-1]
+        end.uniq.reject(&:empty?)
+      end
+
       private
 
-      def redis
-        @redis ||= Redis.connect(::Ost.options)
+      def notify_keep_alive
+        Thread.new do
+          loop do
+            @redis.call 'CLIENT', 'SETNAME', Asynchronic.connection_name
+            sleep Asynchronic.keep_alive_timeout
+          end
+        end
       end
 
 
       class Queue < ::Ost::Queue
 
         def pop
-          key.rpop
+          redis.call 'RPOP', key
         end
 
         def empty?
-          !redis.exists(key)
+          redis.call('EXISTS', key) == 0
         end
 
         def size
@@ -68,7 +76,7 @@ module Asynchronic
         def listen(queue, &block)
           @current_queue = queue
           Asynchronic.retry_execution(self.class, 'listen') do
-            queue.each &block
+            queue.each(&block)
           end
         end
 

data/lib/asynchronic/queue_engine/synchronic.rb

@@ -29,6 +29,11 @@ module Asynchronic
         false
       end
 
+      def active_connections
+        [Asynchronic.connection_name]
+      end
+
+
       class Queue
 
         def initialize(engine)

data/lib/asynchronic/version.rb

@@ -1,3 +1,3 @@
 module Asynchronic
-  VERSION = '1.6.3'
-end
+  VERSION = '2.0.0'
+end

data/lib/asynchronic/worker.rb

@@ -23,7 +23,7 @@ class Asynchronic::Worker
   end
 
   def stop
-    Asynchronic.logger.info('Asynchronic') { "Stopping worker of #{@queue_name} (#{Process.pid})" }
+    Asynchronic.logger.info('Asynchronic') { "Stopping worker of #{queue_name} (#{Process.pid})" }
     listener.stop
   end
 

data/spec/data_store/data_store_examples.rb

@@ -22,19 +22,19 @@ module DataStoreExamples
   end
 
   it 'Delete cascade' do
-    data_store[Key[:key_1]] = 1
-    data_store[Key[:key_1][:key_1_1]] = 2
-    data_store[Key[:key_1][:key_1_2]] = 3
-    data_store[Key[:key_2]] = 4
-    data_store[Key[:key_2][:key_2_1]] = 5
-    data_store[Key[:key_2][:key_2_2]] = 6
+    data_store[Asynchronic::DataStore::Key[:key_1]] = 1
+    data_store[Asynchronic::DataStore::Key[:key_1][:key_1_1]] = 2
+    data_store[Asynchronic::DataStore::Key[:key_1][:key_1_2]] = 3
+    data_store[Asynchronic::DataStore::Key[:key_2]] = 4
+    data_store[Asynchronic::DataStore::Key[:key_2][:key_2_1]] = 5
+    data_store[Asynchronic::DataStore::Key[:key_2][:key_2_2]] = 6
 
-    data_store.delete_cascade Key[:key_1]
+    data_store.delete_cascade Asynchronic::DataStore::Key[:key_1]
 
     data_store.keys.sort.must_equal [
-      Key[:key_2], 
-      Key[:key_2][:key_2_1], 
-      Key[:key_2][:key_2_2]
+      Asynchronic::DataStore::Key[:key_2], 
+      Asynchronic::DataStore::Key[:key_2][:key_2_1], 
+      Asynchronic::DataStore::Key[:key_2][:key_2_2]
     ]
   end
 

data/spec/expectations.rb

@@ -75,6 +75,10 @@ module MiniTest::Assertions
     process.finalized_at.must_be_instance_of Time
   end
 
+  def assert_have_connection_name(process)
+    process.connection_name.must_equal Asynchronic.connection_name
+  end
+
 end
 
 Asynchronic::QueueEngine::InMemory::Queue.infect_an_assertion :assert_enqueued, :must_enqueued
@@ -85,4 +89,5 @@ Asynchronic::Process.infect_an_assertion :assert_be_pending, :must_be_pending, :
 Asynchronic::Process.infect_an_assertion :assert_be_queued, :must_be_queued, :unary
 Asynchronic::Process.infect_an_assertion :assert_be_waiting, :must_be_waiting, :unary
 Asynchronic::Process.infect_an_assertion :assert_be_completed, :must_be_completed, :unary
-Asynchronic::Process.infect_an_assertion :assert_be_aborted, :must_be_aborted, :unary
+Asynchronic::Process.infect_an_assertion :assert_be_aborted, :must_be_aborted, :unary
+Asynchronic::Process.infect_an_assertion :assert_have_connection_name, :must_have_connection_name, :unary

data/spec/facade_spec.rb

@@ -64,4 +64,16 @@ describe Asynchronic, 'Facade' do
     Asynchronic.garbage_collector.must_be_instance_of Asynchronic::GarbageCollector
   end
 
+  it 'Redis data store sync timeout' do
+    Asynchronic.redis_data_store_sync_timeout.must_equal 0.01
+  end
+
+  it 'Keep alive timeout' do
+    Asynchronic.keep_alive_timeout.must_equal 0.1
+  end
+
+  it 'Connection name' do
+    Asynchronic.connection_name.must_equal "HOST=#{Socket.gethostname},PID=#{::Process.pid}"
+  end
+
 end

data/spec/minitest_helper.rb

@@ -5,6 +5,7 @@ require 'minitest/colorin'
 require 'minitest/great_expectations'
 require 'jobs'
 require 'expectations'
+require 'timeout'
 require 'pry-nav'
 
 class Module

data/spec/process/life_cycle_examples.rb

@@ -10,17 +10,22 @@ module LifeCycleExamples
   end
 
   def create(type, params={})
-    env.create_process type, params
+    env.create_process(type, params).tap do |process|
+      process.must_be_initialized
+    end
   end
 
   def execute(queue)
-    env.load_process(queue.pop).execute
+    process = env.load_process(queue.pop)
+    process.execute
+    process.must_have_connection_name
+    process.wont_be :dead?
+    process.send(:connected?).must_be_true
   end
 
   it 'Basic' do
     process = create BasicJob, input: 1
 
-    process.must_be_initialized
     process.must_have_params input: 1
     queue.must_be_empty
 
@@ -39,7 +44,6 @@ module LifeCycleExamples
   it 'Sequential' do
     process = create SequentialJob, input: 50
 
-    process.must_be_initialized
     process.must_have_params input: 50
     queue.must_be_empty
 
@@ -77,7 +81,6 @@ module LifeCycleExamples
   it 'Graph' do
     process = create GraphJob, input: 100
 
-    process.must_be_initialized
     process.must_have_params input: 100
     queue.must_be_empty
 
@@ -133,7 +136,6 @@ module LifeCycleExamples
   it 'Parallel' do
     process = create ParallelJob, input: 10, times: 3
 
-    process.must_be_initialized
     process.must_have_params input: 10, times: 3
     queue.must_be_empty
 
@@ -167,7 +169,6 @@ module LifeCycleExamples
   it 'Nested' do
     process = create NestedJob, input: 4
 
-    process.must_be_initialized
     process.must_have_params input: 4
     queue.must_be_empty
 
@@ -207,7 +208,6 @@ module LifeCycleExamples
   it 'Alias' do
     process = create AliasJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -261,7 +261,6 @@ module LifeCycleExamples
   it 'Custom queue' do
     process = create CustomQueueJob, input: 'hello'
 
-    process.must_be_initialized
     process.must_have_params input: 'hello'
 
     env.queue(:queue_1).must_be_empty
@@ -302,7 +301,6 @@ module LifeCycleExamples
   it 'Exception' do
     process = create ExceptionJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -320,7 +318,6 @@ module LifeCycleExamples
   it 'Inner exception' do
     process = create InnerExceptionJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -348,7 +345,6 @@ module LifeCycleExamples
   it 'Forward reference' do
     process = create ForwardReferenceJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -391,7 +387,6 @@ module LifeCycleExamples
   it 'Job with retries' do
     process = create WithRetriesJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -604,7 +599,7 @@ module LifeCycleExamples
     pid_1 = process_1.id
     pid_2 = process_2.id
 
-    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 37
+    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 38
     data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 7
 
     process_1.destroy
@@ -628,8 +623,8 @@ module LifeCycleExamples
     process_1.must_be_completed
     process_2.must_be_waiting
 
-    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 49
-    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 37
+    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 53
+    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 38
 
     gc = Asynchronic::GarbageCollector.new env, 0.001
     
@@ -651,13 +646,12 @@ module LifeCycleExamples
     gc.start
 
     data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 0
-    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 37
+    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 38
   end
 
   it 'Before finalize hook when completed' do
     process = create BeforeFinalizeCompletedJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -675,7 +669,6 @@ module LifeCycleExamples
   it 'Before finalize hook when aborted' do
     process = create BeforeFinalizeAbortedJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -693,7 +686,6 @@ module LifeCycleExamples
   it 'Before finalize raises exception and aborts' do
     process = create BeforeFinalizeRaisesExceptionJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue
@@ -711,7 +703,6 @@ module LifeCycleExamples
   it 'Before finalize raises exception on aborted job' do
     process = create BeforeFinalizeExceptionOnAbortedJob
 
-    process.must_be_initialized
     queue.must_be_empty
 
     process.enqueue

data/spec/queue_engine/queue_engine_examples.rb

@@ -31,17 +31,23 @@ module QueueEngineExamples
   end
 
   it 'Listener' do
-    queue.push 'msg_1'
-    queue.push 'msg_2'
+    Timeout.timeout(5) do
+      queue.push 'msg_1'
+      queue.push 'msg_2'
 
-    messages = []
+      messages = []
 
-    listener.listen(queue) do |msg|
-      messages << msg
-      listener.stop if queue.empty?
+      listener.listen(queue) do |msg|
+        messages << msg
+        listener.stop if queue.empty?
+      end
+
+      messages.must_equal %w(msg_1 msg_2)
     end
+  end
 
-    messages.must_equal %w(msg_1 msg_2)
+  it 'Active connections' do
+    engine.active_connections.must_equal [Asynchronic.connection_name]
   end
 
 end

data/spec/queue_engine/synchronic_spec.rb

@@ -34,5 +34,9 @@ describe Asynchronic::QueueEngine::Synchronic do
     process.must_be_completed
     process.result.must_equal '10%' => 20, '20%' => 40
   end
-  
+
+  it 'Active connections' do
+    Asynchronic.queue_engine.active_connections.must_equal [Asynchronic.connection_name]
+  end
+
 end

metadata

@@ -1,43 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: asynchronic
 version: !ruby/object:Gem::Version
-  version: 1.6.3
+  version: 2.0.0
 platform: ruby
 authors:
 - Gabriel Naiman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-05 00:00:00.000000000 Z
+date: 2019-06-18 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: ost
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: class_config
   requirement: !ruby/object:Gem::Requirement