asynchronic 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1dfc813a24c2580723d255461a9a6b91376d562
-  data.tar.gz: 793f9fd3f28c7fc712f98adeba227128b37644f9
+  metadata.gz: b2bd845501396b08b9d2948649a8792af6a966cb
+  data.tar.gz: 577a10429b1df9448d84394c85d5f38752f4380b
 SHA512:
-  metadata.gz: 1a003d3fefe9a1a4f79535a6dc0e1639306f347672d75b1aef6e625c59da4aaeca68a98d0d8d73f02646d62bcef6ec431836531d7292bdb25c0d1bd4854a4ff9
-  data.tar.gz: c2d6023bd1ea95d9b6147193bea9be6b36062c5bf774c08cd39e85029e4b5825067902553c14db0be6d05416e82b03ba4bea404376a62c64aad00fd0c0508caa
+  metadata.gz: 58b29093b0856f719a24efc276ab827bc83a492dfa3b781ed3ced23ab6d58e6f869989abd08c1bceb35d94712f060dec419f2d643ba1b83917b8eb0ce538feae
+  data.tar.gz: 7f7248fdc544ac0bd902311ab4893bc02f0545b689206f1061fab5f6e4e0d334dd36c842024fb7bf9c35c764e3d7527c122ddbf7fd9830f05fce3204676e561e

data/Rakefile

@@ -3,7 +3,8 @@ require 'rake/testtask'
 
 Rake::TestTask.new(:spec) do |t|
   t.libs << 'spec'
-  t.pattern = 'spec/**/*_spec.rb'
+  t.libs << 'lib'
+  t.pattern = ENV['DIR'] ? File.join(ENV['DIR'], '**', '*_spec.rb') : 'spec/**/*_spec.rb'
   t.verbose = false
   t.warning = false
   t.loader = nil if ENV['TEST']

data/asynchronic.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'ost', '~> 0.1'
   spec.add_dependency 'class_config', '~> 0.0'
   spec.add_dependency 'transparent_proxy', '~> 0.0'
+  spec.add_dependency 'multi_require', '~> 1.0'
 
   spec.add_development_dependency 'bundler', '~> 1.12'
   spec.add_development_dependency 'rake', '~> 11.0'

data/lib/asynchronic.rb

@@ -5,8 +5,9 @@ require 'ost'
 require 'class_config'
 require 'transparent_proxy'
 require 'logger'
+require 'multi_require'
 
-Dir.glob(File.expand_path('asynchronic/**/*.rb', File.dirname(__FILE__))).sort.each { |f| require f }
+MultiRequire.require_relative_pattern 'asynchronic/**/*.rb'
 
 module Asynchronic
 
@@ -16,9 +17,9 @@ module Asynchronic
   attr_config :queue_engine, QueueEngine::InMemory.new
   attr_config :data_store, DataStore::InMemory.new
   attr_config :logger, Logger.new($stdout)
+  attr_config :retry_timeout, 30
+  attr_config :garbage_collector_timeout, 30
 
-  RETRY_TIMEOUT = 30
-  
   def self.environment
     Environment.new queue_engine, data_store
   end
@@ -31,12 +32,16 @@ module Asynchronic
     environment.processes
   end
 
-  def self.retry_execution(a_class, message)
+  def self.garbage_collector
+    @garbage_collector ||= GarbageCollector.new environment, garbage_collector_timeout
+  end
+
+  def self.retry_execution(klass, message)
     begin
       result = yield
     rescue Exception => ex
-      logger.error(a_class) { "Retrying #{message}. ERROR: #{ex.message}" }
-      sleep RETRY_TIMEOUT
+      logger.error(klass) { "Retrying #{message}. ERROR: #{ex.message}" }
+      sleep retry_timeout
       retry
     end
     result

data/lib/asynchronic/data_store/in_memory.rb

@@ -23,12 +23,17 @@ module Asynchronic
         @hash.delete key.to_s
       end
 
+      def delete_cascade(key)
+        keys = self.keys.select { |k| k.sections.first == key }
+        keys.each { |k| delete k }
+      end
+
       def keys
-        @hash.keys.map { |k| Key.new k }
+        @hash.keys.map { |k| Key[k] }
       end
 
       def synchronize(key, &block)
-        @keys_mutex[key].synchronize &block
+        @keys_mutex[key].synchronize(&block)
       end
 
       def connection_args

data/lib/asynchronic/data_store/key.rb

@@ -3,6 +3,10 @@ module Asynchronic
     class Key < String
     
       SEPARATOR = '|'
+
+      def self.[](key)
+        new key
+      end
       
       def initialize(key)
         super key.to_s

data/lib/asynchronic/data_store/redis.rb

@@ -8,8 +8,8 @@ module Asynchronic
       include Helper
 
       def initialize(scope, *args)
-        @scope = Key.new scope
-        @connection = ::Redis.new *args
+        @scope = Key[scope]
+        @connection = ::Redis.new(*args)
       end
 
       def [](key)
@@ -28,8 +28,13 @@ module Asynchronic
         @connection.del @scope[key]
       end
 
+      def delete_cascade(key)
+        @connection.del @scope[key]
+        @connection.keys(@scope[key]['*']).each { |k| @connection.del k }
+      end
+
       def keys
-        @connection.keys(@scope['*']).map { |k| Key.new(k).remove_first }
+        @connection.keys(@scope['*']).map { |k| Key[k].remove_first }
       end
 
       def synchronize(key)
@@ -46,7 +51,7 @@ module Asynchronic
       end
 
       def self.connect(*args)
-        new *args
+        new(*args)
       end
       
     end

data/lib/asynchronic/data_store/scoped_store.rb

@@ -9,7 +9,7 @@ module Asynchronic
       
       def initialize(data_store, scope)
         @data_store = data_store
-        @scope = Key.new scope
+        @scope = Key[scope]
       end
 
       def [](key)
@@ -24,10 +24,14 @@ module Asynchronic
         @data_store.delete @scope[key]
       end
 
+      def delete_cascade
+        @data_store.delete_cascade @scope
+      end
+
       def keys
         @data_store.keys.
           select { |k| k.start_with? @scope[''] }.
-          map { |k| Key.new(k).remove_first @scope.sections.count }
+          map { |k| Key[k].remove_first @scope.sections.count }
       end
 
       def synchronize(key, &block)

data/lib/asynchronic/garbage_collector.rb

@@ -0,0 +1,61 @@
+module Asynchronic
+  class GarbageCollector
+
+    def initialize(environment, timeout)
+      @environment = environment
+      @timeout = timeout
+      @running = false
+      @conditions = {}
+    end
+
+    def start
+      Asynchronic.logger.info('Asynchronic') { 'Starting GC' }
+
+      Signal.trap('QUIT') { stop }
+
+      @running = true
+
+      while @running
+        processes = environment.processes
+
+        conditions.each do |name, condition|
+          Asynchronic.logger.info('Asynchronic') { "Running GC - #{name}" }
+          begin
+            processes.select(&condition).each(&:destroy)
+          rescue => ex
+            Asynchronic.logger.error('Asynchronic') { "#{ex.class}: #{ex.message}" }
+          end
+        end
+
+        wait
+      end
+    end
+
+    def stop
+      Asynchronic.logger.info('Asynchronic') { 'Stopping GC' }
+      @running = false
+    end
+
+    def add_condition(name, &block)
+      conditions[name] = block
+    end
+
+    def remove_condition(name)
+      conditions.delete name
+    end
+
+    def conditions_names
+      conditions.keys
+    end
+
+    private
+
+    attr_reader :environment, :timeout, :conditions
+
+    def wait
+      Asynchronic.logger.info('Asynchronic') { 'Sleeping GC' }
+      sleep timeout
+    end
+
+  end
+end

data/lib/asynchronic/process.rb

@@ -13,12 +13,14 @@ module Asynchronic
 
     ATTRIBUTE_NAMES = [:type, :name, :queue, :status, :dependencies, :data, :result, :error] | TIME_TRACKING_MAP.values.uniq
 
+    CANCELED_ERROR_MESSAGE = 'Canceled'
+
     attr_reader :id
 
     def initialize(environment, id, &block)
       @environment = environment
-      @id = DataStore::Key.new id
-      instance_eval &block if block_given?
+      @id = DataStore::Key[id]
+      instance_eval(&block) if block_given?
     end
 
     ATTRIBUTE_NAMES.each do |attribute|
@@ -41,6 +43,14 @@ module Asynchronic
       completed? || aborted?
     end
 
+    def cancel!
+      abort! CANCELED_ERROR_MESSAGE
+    end
+
+    def destroy
+      data_store.delete_cascade
+    end
+
     def full_status
       processes.each_with_object(name => status) do |process, hash|
         hash.update(process.full_status)
@@ -60,7 +70,7 @@ module Asynchronic
     end
 
     def [](process_name)
-      processes.detect { |p| p.name == process_name }
+      processes.detect { |p| p.name == process_name.to_s }
     end
 
     def processes
@@ -131,7 +141,7 @@ module Asynchronic
 
       new(environment, id) do
         self.type = type
-        self.name = params.delete(:alias) || type
+        self.name = (params.delete(:alias) || type).to_s
         self.queue = params.delete(:queue) || type.queue || parent_queue
         self.dependencies = Array(params.delete(:dependencies)) | Array(params.delete(:dependency)) | infer_dependencies(params)
         self.params = params

data/lib/asynchronic/version.rb

@@ -1,3 +1,3 @@
 module Asynchronic
-  VERSION = '1.4.0'
+  VERSION = '1.5.0'
 end

data/spec/data_store/data_store_examples.rb

@@ -21,6 +21,23 @@ module DataStoreExamples
     data_store[:key].must_be_nil
   end
 
+  it 'Delete cascade' do
+    data_store[Key[:key_1]] = 1
+    data_store[Key[:key_1][:key_1_1]] = 2
+    data_store[Key[:key_1][:key_1_2]] = 3
+    data_store[Key[:key_2]] = 4
+    data_store[Key[:key_2][:key_2_1]] = 5
+    data_store[Key[:key_2][:key_2_2]] = 6
+
+    data_store.delete_cascade Key[:key_1]
+
+    data_store.keys.sort.must_equal [
+      Key[:key_2], 
+      Key[:key_2][:key_2_1], 
+      Key[:key_2][:key_2_2]
+    ]
+  end
+
   it 'Each' do
     data_store[:a] = 1
     data_store[:b] = 2

data/spec/data_store/key_spec.rb

@@ -5,54 +5,54 @@ describe Asynchronic::DataStore::Key do
   Key = Asynchronic::DataStore::Key
 
   it 'Return the namespace' do
-    key = Key.new 'foo'
+    key = Key['foo']
     key.must_equal 'foo'
   end
 
   it 'Prepend the namespace' do
-    key = Key.new 'foo'
+    key = Key['foo']
     key['bar'].must_equal 'foo|bar'
   end
 
   it 'Work in more than one level' do
-    key_1 = Key.new 'foo'
-    key_2 = Key.new key_1['bar']
+    key_1 = Key['foo']
+    key_2 = Key[key_1['bar']]
     key_2['baz'].must_equal 'foo|bar|baz'
   end
 
   it 'Be chainable' do
-    key = Key.new 'foo'
+    key = Key['foo']
     key['bar']['baz'].must_equal 'foo|bar|baz'
   end
 
   it 'Accept symbols' do
-    key = Key.new :foo
+    key = Key[:foo]
     key[:bar].must_equal 'foo|bar'
   end
 
   it 'Accept numbers' do
-    key = Key.new 'foo'
+    key = Key['foo']
     key[3].must_equal 'foo|3'
   end
 
   it 'Split in sections' do
-    key = Key.new(:foo)[:bar][:buz]
+    key = Key[:foo][:bar][:buz]
     key.sections.must_equal %w(foo bar buz)
   end
 
   it 'Detect nested sections' do
-    Key.new(:foo).wont_be :nested?
-    Key.new(:foo)[:bar].must_be :nested?
+    Key[:foo].wont_be :nested?
+    Key[:foo][:bar].must_be :nested?
   end
 
   it 'Remove first sections' do
-    key = Key.new(:foo)[:bar][:buz]
+    key = Key[:foo][:bar][:buz]
     key.remove_first.must_equal 'bar|buz'
     key.remove_first(2).must_equal 'buz'
   end
 
   it 'Remove last sections' do
-    key = Key.new(:foo)[:bar][:buz]
+    key = Key[:foo][:bar][:buz]
     key.remove_last.must_equal 'foo|bar'
     key.remove_last(2).must_equal 'foo'
   end

data/spec/facade_spec.rb

@@ -60,4 +60,8 @@ describe Asynchronic, 'Facade' do
     end
   end
 
+  it 'Garbage collector' do
+    Asynchronic.garbage_collector.must_be_instance_of Asynchronic::GarbageCollector
+  end
+
 end

data/spec/process/life_cycle_examples.rb

@@ -493,73 +493,165 @@ module LifeCycleExamples
 
     execute queue
 
-    process.full_status.must_equal AbortQueuedAfertErrorJob => :waiting,
-                                   AbortQueuedAfertErrorJob::Child_1 => :queued,
-                                   AbortQueuedAfertErrorJob::Child_2 => :queued,
-                                   AbortQueuedAfertErrorJob::Child_3 => :queued,
-                                   AbortQueuedAfertErrorJob::Child_4 => :queued
+    process.full_status.must_equal 'AbortQueuedAfertErrorJob'          => :waiting,
+                                   'AbortQueuedAfertErrorJob::Child_1' => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_2' => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_3' => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_4' => :queued
 
     execute queue
 
-    process.full_status.must_equal AbortQueuedAfertErrorJob => :waiting,
-                                   AbortQueuedAfertErrorJob::Child_1 => :waiting,
-                                   'Child_1_1' => :queued,
-                                   'Child_1_2' => :queued,
-                                   AbortQueuedAfertErrorJob::Child_2 => :queued,
-                                   AbortQueuedAfertErrorJob::Child_3 => :queued,
-                                   AbortQueuedAfertErrorJob::Child_4 => :queued
+    process.full_status.must_equal 'AbortQueuedAfertErrorJob'          => :waiting,
+                                   'AbortQueuedAfertErrorJob::Child_1' => :waiting,
+                                   'Child_1_1'                         => :queued,
+                                   'Child_1_2'                         => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_2' => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_3' => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_4' => :queued
 
     execute queue
 
-    process.full_status.must_equal AbortQueuedAfertErrorJob => :waiting,
-                                   AbortQueuedAfertErrorJob::Child_1 => :waiting,
-                                   'Child_1_1' => :queued,
-                                   'Child_1_2' => :queued,
-                                   AbortQueuedAfertErrorJob::Child_2 => :completed,
-                                   AbortQueuedAfertErrorJob::Child_3 => :queued,
-                                   AbortQueuedAfertErrorJob::Child_4 => :queued
+    process.full_status.must_equal 'AbortQueuedAfertErrorJob'          => :waiting,
+                                   'AbortQueuedAfertErrorJob::Child_1' => :waiting,
+                                   'Child_1_1'                         => :queued,
+                                   'Child_1_2'                         => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_2' => :completed,
+                                   'AbortQueuedAfertErrorJob::Child_3' => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_4' => :queued
 
     execute queue
 
-    process.full_status.must_equal AbortQueuedAfertErrorJob => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_1 => :waiting,
-                                   'Child_1_1' => :queued,
-                                   'Child_1_2' => :queued,
-                                   AbortQueuedAfertErrorJob::Child_2 => :completed,
-                                   AbortQueuedAfertErrorJob::Child_3 => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_4 => :queued
+    process.full_status.must_equal 'AbortQueuedAfertErrorJob'          => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_1' => :waiting,
+                                   'Child_1_1'                         => :queued,
+                                   'Child_1_2'                         => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_2' => :completed,
+                                   'AbortQueuedAfertErrorJob::Child_3' => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_4' => :queued
 
     execute queue
 
-    process.full_status.must_equal AbortQueuedAfertErrorJob => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_1 => :waiting,
-                                   'Child_1_1' => :queued,
-                                   'Child_1_2' => :queued,
-                                   AbortQueuedAfertErrorJob::Child_2 => :completed,
-                                   AbortQueuedAfertErrorJob::Child_3 => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_4 => :aborted
+    process.full_status.must_equal 'AbortQueuedAfertErrorJob'          => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_1' => :waiting,
+                                   'Child_1_1'                         => :queued,
+                                   'Child_1_2'                         => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_2' => :completed,
+                                   'AbortQueuedAfertErrorJob::Child_3' => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_4' => :aborted
 
     execute queue
 
-    process.full_status.must_equal AbortQueuedAfertErrorJob => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_1 => :aborted,
-                                   'Child_1_1' => :aborted,
-                                   'Child_1_2' => :queued,
-                                   AbortQueuedAfertErrorJob::Child_2 => :completed,
-                                   AbortQueuedAfertErrorJob::Child_3 => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_4 => :aborted
+    process.full_status.must_equal 'AbortQueuedAfertErrorJob'          => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_1' => :aborted,
+                                   'Child_1_1'                         => :aborted,
+                                   'Child_1_2'                         => :queued,
+                                   'AbortQueuedAfertErrorJob::Child_2' => :completed,
+                                   'AbortQueuedAfertErrorJob::Child_3' => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_4' => :aborted
 
     execute queue
 
-    process.full_status.must_equal AbortQueuedAfertErrorJob => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_1 => :aborted,
-                                   'Child_1_1' => :aborted,
-                                   'Child_1_2' => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_2 => :completed,
-                                   AbortQueuedAfertErrorJob::Child_3 => :aborted,
-                                   AbortQueuedAfertErrorJob::Child_4 => :aborted
+    process.full_status.must_equal 'AbortQueuedAfertErrorJob'          => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_1' => :aborted,
+                                   'Child_1_1'                         => :aborted,
+                                   'Child_1_2'                         => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_2' => :completed,
+                                   'AbortQueuedAfertErrorJob::Child_3' => :aborted,
+                                   'AbortQueuedAfertErrorJob::Child_4' => :aborted
 
     process.real_error.must_equal 'Forced error'
   end
 
+  it 'Manual abort' do
+    process = create NestedJob, input: 10
+
+    process.enqueue
+
+    execute queue
+
+    process.full_status.must_equal 'NestedJob'         => :waiting, 
+                                   'NestedJob::Level1' => :queued
+
+    execute queue
+
+    process.full_status.must_equal 'NestedJob'                 => :waiting, 
+                                   'NestedJob::Level1'         => :waiting, 
+                                   'NestedJob::Level1::Level2' => :queued
+
+    process.cancel!
+
+    process.real_error.must_equal Asynchronic::Process::CANCELED_ERROR_MESSAGE
+
+    process.full_status.must_equal 'NestedJob'                 => :aborted, 
+                                   'NestedJob::Level1'         => :waiting, 
+                                   'NestedJob::Level1::Level2' => :queued
+
+    execute queue
+
+    process.full_status.must_equal 'NestedJob'                 => :aborted, 
+                                   'NestedJob::Level1'         => :aborted, 
+                                   'NestedJob::Level1::Level2' => :aborted
+  end
+
+  it 'Remove process' do
+    process_1 = create AliasJob
+    process_2 = create AliasJob
+
+    process_1.enqueue
+
+    execute queue
+
+    pid_1 = process_1.id
+    pid_2 = process_2.id
+
+    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 37
+    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 7
+
+    process_1.destroy
+
+    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 0
+    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 7
+  end
+
+  it 'Garbage collector' do
+    process_1 = create AliasJob
+    process_1.enqueue
+    4.times { execute queue }
+    
+    process_2 = create AliasJob
+    process_2.enqueue
+    execute queue
+
+    pid_1 = process_1.id
+    pid_2 = process_2.id
+
+    process_1.must_be_completed
+    process_2.must_be_waiting
+
+    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 49
+    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 37
+
+    gc = Asynchronic::GarbageCollector.new env, 0.001
+    
+    gc.add_condition('Completed', &:completed?)
+    gc.add_condition('Waiting', &:waiting?)
+    gc.add_condition('Exception') { raise 'Invalid condition' }
+
+    gc.conditions_names.must_equal ['Completed', 'Waiting', 'Exception']
+
+    gc.remove_condition 'Waiting'
+    
+    gc.conditions_names.must_equal ['Completed', 'Exception']
+
+    Thread.new do
+      sleep 0.01
+      gc.stop
+    end
+
+    gc.start
+
+    data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 0
+    data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 37
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: asynchronic
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Gabriel Naiman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-20 00:00:00.000000000 Z
+date: 2018-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
+- !ruby/object:Gem::Dependency
+  name: multi_require
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -221,6 +235,7 @@ files:
 - lib/asynchronic/data_store/scoped_store.rb
 - lib/asynchronic/environment.rb
 - lib/asynchronic/error.rb
+- lib/asynchronic/garbage_collector.rb
 - lib/asynchronic/job.rb
 - lib/asynchronic/process.rb
 - lib/asynchronic/queue_engine/in_memory.rb