asynchronic 2.0.0 → 3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +6 -9
- data/README.md +0 -1
- data/asynchronic.gemspec +3 -2
- data/lib/asynchronic.rb +5 -3
- data/lib/asynchronic/data_store/redis.rb +8 -8
- data/lib/asynchronic/environment.rb +3 -3
- data/lib/asynchronic/garbage_collector.rb +2 -0
- data/lib/asynchronic/notifier/broadcaster.rb +30 -0
- data/lib/asynchronic/notifier/in_memory.rb +33 -0
- data/lib/asynchronic/process.rb +19 -8
- data/lib/asynchronic/queue_engine/ost.rb +16 -10
- data/lib/asynchronic/version.rb +1 -1
- data/spec/data_store/lazy_value_examples.rb +0 -1
- data/spec/facade_spec.rb +2 -2
- data/spec/jobs.rb +2 -2
- data/spec/minitest_helper.rb +1 -0
- data/spec/process/life_cycle_examples.rb +88 -43
- data/spec/process/life_cycle_in_memory_spec.rb +1 -0
- data/spec/process/life_cycle_redis_spec.rb +1 -0
- data/spec/queue_engine/ost_spec.rb +4 -0
- data/spec/worker/in_memory_spec.rb +1 -0
- data/spec/worker/redis_spec.rb +1 -0
- data/spec/worker/worker_examples.rb +1 -1
- metadata +38 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7cc53216188aca01f66806e933e68d3edacda3dabca5e93b3da3eae1a41bbae5
|
4
|
+
data.tar.gz: ad209146fc5b92f5d0d38c5d0ce27b72b6c6f7308831f9facb08a9bc7a538904
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3e826e042a938689168e9ca085d4fd5f28250ed350ebdb658c871415b36df88088e290712c72558256c1535b247ead901fbae3818e692895f6e4d8052ab5f560
|
7
|
+
data.tar.gz: 2a1d517fb7cd29556d790e6680928e9294e79329b23a3e7ebab6e948ebc6a087d713cc2c3ca12908ca25f4798585198d35a88351c3f0723ad47d8e4b4416bb2d
|
data/.travis.yml
CHANGED
@@ -4,11 +4,12 @@ rvm:
|
|
4
4
|
- 2.0
|
5
5
|
- 2.1
|
6
6
|
- 2.2
|
7
|
-
- 2.3
|
8
|
-
- 2.4
|
9
|
-
- 2.5
|
10
|
-
- 2.6
|
11
|
-
-
|
7
|
+
- 2.3
|
8
|
+
- 2.4
|
9
|
+
- 2.5
|
10
|
+
- 2.6
|
11
|
+
- 2.7
|
12
|
+
- jruby-9.2.9.0
|
12
13
|
- ruby-head
|
13
14
|
- jruby-head
|
14
15
|
|
@@ -18,9 +19,5 @@ matrix:
|
|
18
19
|
- rvm: ruby-head
|
19
20
|
- rvm: jruby-head
|
20
21
|
|
21
|
-
before_install:
|
22
|
-
- rvm all-gemsets do gem uninstall bundler -ax || true
|
23
|
-
- gem install bundler -v "< 2"
|
24
|
-
|
25
22
|
services:
|
26
23
|
- redis-server
|
data/README.md
CHANGED
@@ -4,7 +4,6 @@
|
|
4
4
|
[](https://travis-ci.org/gabynaiman/asynchronic)
|
5
5
|
[](https://coveralls.io/r/gabynaiman/asynchronic?branch=master)
|
6
6
|
[](https://codeclimate.com/github/gabynaiman/asynchronic)
|
7
|
-
[](https://gemnasium.com/gabynaiman/asynchronic)
|
8
7
|
|
9
8
|
DSL for asynchronic pipeline using queues over Redis
|
10
9
|
|
data/asynchronic.gemspec
CHANGED
@@ -19,14 +19,15 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
21
|
spec.add_dependency 'ost', '~> 1.0'
|
22
|
+
spec.add_dependency 'broadcaster', '~> 1.0', '>= 1.0.2'
|
22
23
|
spec.add_dependency 'class_config', '~> 0.0'
|
23
24
|
spec.add_dependency 'transparent_proxy', '~> 0.0'
|
24
25
|
spec.add_dependency 'multi_require', '~> 1.0'
|
25
26
|
|
26
|
-
spec.add_development_dependency '
|
27
|
-
spec.add_development_dependency 'rake', '~> 11.0'
|
27
|
+
spec.add_development_dependency 'rake', '~> 12.0'
|
28
28
|
spec.add_development_dependency 'minitest', '~> 5.0', '< 5.11'
|
29
29
|
spec.add_development_dependency 'minitest-great_expectations', '~> 0.0'
|
30
|
+
spec.add_development_dependency 'minitest-stub_any_instance', '~> 1.0'
|
30
31
|
spec.add_development_dependency 'minitest-colorin', '~> 0.1'
|
31
32
|
spec.add_development_dependency 'minitest-line', '~> 0.6'
|
32
33
|
spec.add_development_dependency 'simplecov', '~> 0.12'
|
data/lib/asynchronic.rb
CHANGED
@@ -2,6 +2,7 @@ require 'forwardable'
|
|
2
2
|
require 'securerandom'
|
3
3
|
require 'ost'
|
4
4
|
require 'redic'
|
5
|
+
require 'broadcaster'
|
5
6
|
require 'class_config'
|
6
7
|
require 'transparent_proxy'
|
7
8
|
require 'logger'
|
@@ -18,15 +19,16 @@ module Asynchronic
|
|
18
19
|
attr_config :default_queue, :asynchronic
|
19
20
|
attr_config :queue_engine, QueueEngine::InMemory.new
|
20
21
|
attr_config :data_store, DataStore::InMemory.new
|
22
|
+
attr_config :notifier, Notifier::InMemory.new
|
21
23
|
attr_config :logger, Logger.new($stdout)
|
22
24
|
attr_config :retry_timeout, 30
|
23
25
|
attr_config :garbage_collector_timeout, 30
|
24
26
|
attr_config :redis_data_store_sync_timeout, 0.01
|
25
|
-
attr_config :keep_alive_timeout,
|
26
|
-
attr_config :connection_name, "HOST=#{Socket.gethostname},PID=#{::Process.pid}"
|
27
|
+
attr_config :keep_alive_timeout, 1
|
28
|
+
attr_config :connection_name, "HOST=#{Socket.gethostname},PID=#{::Process.pid},UUID=#{SecureRandom.uuid}"
|
27
29
|
|
28
30
|
def self.environment
|
29
|
-
Environment.new queue_engine, data_store
|
31
|
+
Environment.new queue_engine, data_store, notifier
|
30
32
|
end
|
31
33
|
|
32
34
|
def self.[](pid)
|
@@ -12,7 +12,7 @@ module Asynchronic
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def [](key)
|
15
|
-
value = @redis.call 'GET', @scope[key]
|
15
|
+
value = @redis.call! 'GET', @scope[key]
|
16
16
|
value ? Marshal.load(value) : nil
|
17
17
|
rescue => ex
|
18
18
|
Asynchronic.logger.warn('Asynchronic') { ex.message }
|
@@ -20,29 +20,29 @@ module Asynchronic
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def []=(key, value)
|
23
|
-
@redis.call 'SET', @scope[key], Marshal.dump(value)
|
23
|
+
@redis.call! 'SET', @scope[key], Marshal.dump(value)
|
24
24
|
end
|
25
25
|
|
26
26
|
def delete(key)
|
27
|
-
@redis.call 'DEL', @scope[key]
|
27
|
+
@redis.call! 'DEL', @scope[key]
|
28
28
|
end
|
29
29
|
|
30
30
|
def delete_cascade(key)
|
31
|
-
@redis.call 'DEL', @scope[key]
|
32
|
-
@redis.call('KEYS', @scope[key]['*']).each { |k| @redis.call 'DEL', k }
|
31
|
+
@redis.call! 'DEL', @scope[key]
|
32
|
+
@redis.call!('KEYS', @scope[key]['*']).each { |k| @redis.call! 'DEL', k }
|
33
33
|
end
|
34
34
|
|
35
35
|
def keys
|
36
|
-
@redis.call('KEYS', @scope['*']).map { |k| Key[k].remove_first }
|
36
|
+
@redis.call!('KEYS', @scope['*']).map { |k| Key[k].remove_first }
|
37
37
|
end
|
38
38
|
|
39
39
|
def synchronize(key)
|
40
|
-
while @redis.call('GETSET', @scope[key][LOCKED], LOCKED) == LOCKED
|
40
|
+
while @redis.call!('GETSET', @scope[key][LOCKED], LOCKED) == LOCKED
|
41
41
|
sleep Asynchronic.redis_data_store_sync_timeout
|
42
42
|
end
|
43
43
|
yield
|
44
44
|
ensure
|
45
|
-
@redis.call 'DEL', @scope[key][LOCKED]
|
45
|
+
@redis.call! 'DEL', @scope[key][LOCKED]
|
46
46
|
end
|
47
47
|
|
48
48
|
def connection_args
|
@@ -1,12 +1,12 @@
|
|
1
1
|
module Asynchronic
|
2
2
|
class Environment
|
3
3
|
|
4
|
-
attr_reader :queue_engine
|
5
|
-
attr_reader :data_store
|
4
|
+
attr_reader :queue_engine, :data_store, :notifier
|
6
5
|
|
7
|
-
def initialize(queue_engine, data_store)
|
6
|
+
def initialize(queue_engine, data_store, notifier)
|
8
7
|
@queue_engine = queue_engine
|
9
8
|
@data_store = data_store
|
9
|
+
@notifier = notifier
|
10
10
|
end
|
11
11
|
|
12
12
|
def queue(name)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Asynchronic
|
2
|
+
module Notifier
|
3
|
+
class Broadcaster
|
4
|
+
|
5
|
+
def initialize(options={})
|
6
|
+
options[:logger] ||= Asynchronic.logger
|
7
|
+
@broadcaster = ::Broadcaster.new options
|
8
|
+
end
|
9
|
+
|
10
|
+
def publish(pid, event, data=nil)
|
11
|
+
@broadcaster.publish DataStore::Key[pid][event], data
|
12
|
+
end
|
13
|
+
|
14
|
+
def subscribe(pid, event, &block)
|
15
|
+
@broadcaster.subscribe DataStore::Key[pid][event] do |data|
|
16
|
+
block.call data
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def unsubscribe(subscription_id)
|
21
|
+
@broadcaster.unsubscribe subscription_id
|
22
|
+
end
|
23
|
+
|
24
|
+
def unsubscribe_all
|
25
|
+
@broadcaster.unsubscribe_all
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Asynchronic
|
2
|
+
module Notifier
|
3
|
+
class InMemory
|
4
|
+
|
5
|
+
def publish(pid, event, data=nil)
|
6
|
+
subscriptions[DataStore::Key[pid][event]].each_value do |block|
|
7
|
+
block.call data
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def subscribe(pid, event, &block)
|
12
|
+
SecureRandom.uuid.tap do |subscription_id|
|
13
|
+
subscriptions[DataStore::Key[pid][event]][subscription_id] = block
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def unsubscribe(subscription_id)
|
18
|
+
subscriptions.each_value { |s| s.delete subscription_id }
|
19
|
+
end
|
20
|
+
|
21
|
+
def unsubscribe_all
|
22
|
+
subscriptions.clear
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def subscriptions
|
28
|
+
@subscriptions ||= Hash.new { |h,k| h[k] = {} }
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/asynchronic/process.rb
CHANGED
@@ -13,7 +13,9 @@ module Asynchronic
|
|
13
13
|
|
14
14
|
ATTRIBUTE_NAMES = [:type, :name, :queue, :status, :dependencies, :data, :result, :error, :connection_name] | TIME_TRACKING_MAP.values.uniq
|
15
15
|
|
16
|
+
AUTOMATIC_ABORTED_ERROR_MESSAGE = 'Automatic aborted before execution'
|
16
17
|
CANCELED_ERROR_MESSAGE = 'Canceled'
|
18
|
+
DEAD_ERROR_MESSAGE = 'Process connection broken'
|
17
19
|
|
18
20
|
attr_reader :id
|
19
21
|
|
@@ -51,6 +53,10 @@ module Asynchronic
|
|
51
53
|
(running? && !connected?) || processes.any?(&:dead?)
|
52
54
|
end
|
53
55
|
|
56
|
+
def abort_if_dead
|
57
|
+
abort! DEAD_ERROR_MESSAGE if dead?
|
58
|
+
end
|
59
|
+
|
54
60
|
def destroy
|
55
61
|
data_store.delete_cascade
|
56
62
|
end
|
@@ -92,13 +98,11 @@ module Asynchronic
|
|
92
98
|
end
|
93
99
|
|
94
100
|
def real_error
|
95
|
-
return nil
|
101
|
+
return nil if !aborted?
|
96
102
|
|
97
|
-
processes.
|
98
|
-
return child.real_error if child.error
|
99
|
-
end
|
103
|
+
first_aborted_child = processes.select(&:aborted?).sort_by(&:finalized_at).first
|
100
104
|
|
101
|
-
error.message
|
105
|
+
first_aborted_child ? first_aborted_child.real_error : error.message
|
102
106
|
end
|
103
107
|
|
104
108
|
def dependencies
|
@@ -190,8 +194,12 @@ module Asynchronic
|
|
190
194
|
|
191
195
|
def status=(status)
|
192
196
|
Asynchronic.logger.info('Asynchronic') { "#{status.to_s.capitalize} #{type} (#{id})" }
|
197
|
+
|
193
198
|
data_store[:status] = status
|
194
199
|
data_store[TIME_TRACKING_MAP[status]] = Time.now if TIME_TRACKING_MAP.key? status
|
200
|
+
|
201
|
+
environment.notifier.publish id, :status_changed, status
|
202
|
+
environment.notifier.publish id, :finalized if finalized?
|
195
203
|
end
|
196
204
|
|
197
205
|
STATUSES.each do |status|
|
@@ -206,8 +214,8 @@ module Asynchronic
|
|
206
214
|
end
|
207
215
|
end
|
208
216
|
|
209
|
-
def abort!(exception
|
210
|
-
self.error = Error.new exception
|
217
|
+
def abort!(exception)
|
218
|
+
self.error = Error.new exception
|
211
219
|
aborted!
|
212
220
|
end
|
213
221
|
|
@@ -215,7 +223,7 @@ module Asynchronic
|
|
215
223
|
self.connection_name = Asynchronic.connection_name
|
216
224
|
|
217
225
|
if root.aborted?
|
218
|
-
abort!
|
226
|
+
abort! AUTOMATIC_ABORTED_ERROR_MESSAGE
|
219
227
|
else
|
220
228
|
running!
|
221
229
|
self.result = job.call
|
@@ -258,6 +266,9 @@ module Asynchronic
|
|
258
266
|
|
259
267
|
def connected?
|
260
268
|
connection_name && environment.queue_engine.active_connections.include?(connection_name)
|
269
|
+
rescue => ex
|
270
|
+
Asynchronic.logger.error('Asynchronic') { "#{ex.message}\n#{ex.backtrace.join("\n")}" }
|
271
|
+
true
|
261
272
|
end
|
262
273
|
|
263
274
|
end
|
@@ -2,12 +2,12 @@ module Asynchronic
|
|
2
2
|
module QueueEngine
|
3
3
|
class Ost
|
4
4
|
|
5
|
-
attr_reader :default_queue
|
5
|
+
attr_reader :redis, :default_queue
|
6
6
|
|
7
7
|
def initialize(options={})
|
8
8
|
@redis = Redic.new(*Array(options[:redis]))
|
9
9
|
@default_queue = options.fetch(:default_queue, Asynchronic.default_queue)
|
10
|
-
@queues ||= Hash.new { |h,k| h[k] = Queue.new k }
|
10
|
+
@queues ||= Hash.new { |h,k| h[k] = Queue.new k, redis }
|
11
11
|
@keep_alive_thread = notify_keep_alive
|
12
12
|
end
|
13
13
|
|
@@ -16,12 +16,12 @@ module Asynchronic
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def queues
|
19
|
-
(@queues.values.map(&:key) |
|
19
|
+
(@queues.values.map(&:key) | redis.call!('KEYS', 'ost:*')).map { |q| q.to_s[4..-1].to_sym }
|
20
20
|
end
|
21
21
|
|
22
22
|
def clear
|
23
23
|
@queues.clear
|
24
|
-
|
24
|
+
redis.call!('KEYS', 'ost:*').each { |k| redis.call!('DEL', k) }
|
25
25
|
end
|
26
26
|
|
27
27
|
def listener
|
@@ -33,9 +33,10 @@ module Asynchronic
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def active_connections
|
36
|
-
|
37
|
-
connection_info.split(' ').detect { |a| a.match(/name=/) }
|
38
|
-
|
36
|
+
redis.call!('CLIENT', 'LIST').split("\n").map do |connection_info|
|
37
|
+
name_attr = connection_info.split(' ').detect { |a| a.match(/name=/) }
|
38
|
+
name_attr ? name_attr[5..-1] : nil
|
39
|
+
end.uniq.compact.reject(&:empty?)
|
39
40
|
end
|
40
41
|
|
41
42
|
private
|
@@ -43,7 +44,7 @@ module Asynchronic
|
|
43
44
|
def notify_keep_alive
|
44
45
|
Thread.new do
|
45
46
|
loop do
|
46
|
-
|
47
|
+
redis.call! 'CLIENT', 'SETNAME', Asynchronic.connection_name
|
47
48
|
sleep Asynchronic.keep_alive_timeout
|
48
49
|
end
|
49
50
|
end
|
@@ -52,12 +53,17 @@ module Asynchronic
|
|
52
53
|
|
53
54
|
class Queue < ::Ost::Queue
|
54
55
|
|
56
|
+
def initialize(name, redis)
|
57
|
+
super name
|
58
|
+
self.redis = redis
|
59
|
+
end
|
60
|
+
|
55
61
|
def pop
|
56
|
-
redis.call 'RPOP', key
|
62
|
+
redis.call! 'RPOP', key
|
57
63
|
end
|
58
64
|
|
59
65
|
def empty?
|
60
|
-
redis.call('EXISTS', key) == 0
|
66
|
+
redis.call!('EXISTS', key) == 0
|
61
67
|
end
|
62
68
|
|
63
69
|
def size
|
data/lib/asynchronic/version.rb
CHANGED
data/spec/facade_spec.rb
CHANGED
@@ -69,11 +69,11 @@ describe Asynchronic, 'Facade' do
|
|
69
69
|
end
|
70
70
|
|
71
71
|
it 'Keep alive timeout' do
|
72
|
-
Asynchronic.keep_alive_timeout.must_equal
|
72
|
+
Asynchronic.keep_alive_timeout.must_equal 1
|
73
73
|
end
|
74
74
|
|
75
75
|
it 'Connection name' do
|
76
|
-
Asynchronic.connection_name.
|
76
|
+
Asynchronic.connection_name.must_match /^HOST=#{Socket.gethostname},PID=#{::Process.pid}/
|
77
77
|
end
|
78
78
|
|
79
79
|
end
|
data/spec/jobs.rb
CHANGED
@@ -200,7 +200,7 @@ end
|
|
200
200
|
class WithRetriesJob < Asynchronic::Job
|
201
201
|
def call
|
202
202
|
@counter = 0
|
203
|
-
retry_when [RuntimeError] do
|
203
|
+
retry_when [RuntimeError], 0.1 do
|
204
204
|
@counter += 1
|
205
205
|
raise 'Counter < 3' if @counter < 3
|
206
206
|
@counter
|
@@ -326,7 +326,7 @@ class NestedJobWithErrorInChildJob < Asynchronic::Job
|
|
326
326
|
end
|
327
327
|
|
328
328
|
|
329
|
-
class
|
329
|
+
class AbortQueuedAfterErrorJob < Asynchronic::Job
|
330
330
|
def call
|
331
331
|
async Child_1
|
332
332
|
async Child_2
|
data/spec/minitest_helper.rb
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
module LifeCycleExamples
|
2
2
|
|
3
|
-
let(:env) { Asynchronic::Environment.new queue_engine, data_store }
|
3
|
+
let(:env) { Asynchronic::Environment.new queue_engine, data_store, notifier }
|
4
4
|
|
5
5
|
let(:queue) { env.default_queue }
|
6
6
|
|
7
7
|
after do
|
8
8
|
data_store.clear
|
9
9
|
queue_engine.clear
|
10
|
+
notifier.unsubscribe_all
|
10
11
|
end
|
11
12
|
|
12
13
|
def create(type, params={})
|
@@ -17,10 +18,46 @@ module LifeCycleExamples
|
|
17
18
|
|
18
19
|
def execute(queue)
|
19
20
|
process = env.load_process(queue.pop)
|
21
|
+
|
22
|
+
events = []
|
23
|
+
status_changed_id = notifier.subscribe(process.id, :status_changed) { |status| events << status }
|
24
|
+
|
25
|
+
is_finalized = false
|
26
|
+
finalized_id = notifier.subscribe(process.id, :finalized) { is_finalized = true }
|
27
|
+
|
20
28
|
process.execute
|
29
|
+
|
21
30
|
process.must_have_connection_name
|
22
31
|
process.wont_be :dead?
|
32
|
+
|
23
33
|
process.send(:connected?).must_be_true
|
34
|
+
|
35
|
+
env.queue_engine.stub(:active_connections, ->() { raise 'Forced error' }) do
|
36
|
+
process.send(:connected?).must_be_true
|
37
|
+
end
|
38
|
+
|
39
|
+
with_retries do
|
40
|
+
events.last.must_equal process.status
|
41
|
+
process.finalized?.must_equal is_finalized
|
42
|
+
end
|
43
|
+
|
44
|
+
notifier.unsubscribe status_changed_id
|
45
|
+
notifier.unsubscribe finalized_id
|
46
|
+
|
47
|
+
status = process.status
|
48
|
+
process.abort_if_dead
|
49
|
+
process.status.must_equal status
|
50
|
+
end
|
51
|
+
|
52
|
+
def with_retries(&block)
|
53
|
+
Timeout.timeout(3) do
|
54
|
+
begin
|
55
|
+
block.call
|
56
|
+
rescue Minitest::Assertion
|
57
|
+
sleep 0.001
|
58
|
+
retry
|
59
|
+
end
|
60
|
+
end
|
24
61
|
end
|
25
62
|
|
26
63
|
it 'Basic' do
|
@@ -481,80 +518,81 @@ module LifeCycleExamples
|
|
481
518
|
process.real_error.must_equal "Error in parent"
|
482
519
|
end
|
483
520
|
|
484
|
-
it 'Abort queued
|
485
|
-
process = create
|
521
|
+
it 'Abort queued After error' do
|
522
|
+
process = create AbortQueuedAfterErrorJob
|
486
523
|
|
487
524
|
process.enqueue
|
488
525
|
|
489
526
|
execute queue
|
490
527
|
|
491
|
-
process.full_status.must_equal '
|
492
|
-
'
|
493
|
-
'
|
494
|
-
'
|
495
|
-
'
|
528
|
+
process.full_status.must_equal 'AbortQueuedAfterErrorJob' => :waiting,
|
529
|
+
'AbortQueuedAfterErrorJob::Child_1' => :queued,
|
530
|
+
'AbortQueuedAfterErrorJob::Child_2' => :queued,
|
531
|
+
'AbortQueuedAfterErrorJob::Child_3' => :queued,
|
532
|
+
'AbortQueuedAfterErrorJob::Child_4' => :queued
|
496
533
|
|
497
534
|
execute queue
|
498
535
|
|
499
|
-
process.full_status.must_equal '
|
500
|
-
'
|
536
|
+
process.full_status.must_equal 'AbortQueuedAfterErrorJob' => :waiting,
|
537
|
+
'AbortQueuedAfterErrorJob::Child_1' => :waiting,
|
501
538
|
'Child_1_1' => :queued,
|
502
539
|
'Child_1_2' => :queued,
|
503
|
-
'
|
504
|
-
'
|
505
|
-
'
|
540
|
+
'AbortQueuedAfterErrorJob::Child_2' => :queued,
|
541
|
+
'AbortQueuedAfterErrorJob::Child_3' => :queued,
|
542
|
+
'AbortQueuedAfterErrorJob::Child_4' => :queued
|
506
543
|
|
507
544
|
execute queue
|
508
545
|
|
509
|
-
process.full_status.must_equal '
|
510
|
-
'
|
546
|
+
process.full_status.must_equal 'AbortQueuedAfterErrorJob' => :waiting,
|
547
|
+
'AbortQueuedAfterErrorJob::Child_1' => :waiting,
|
511
548
|
'Child_1_1' => :queued,
|
512
549
|
'Child_1_2' => :queued,
|
513
|
-
'
|
514
|
-
'
|
515
|
-
'
|
550
|
+
'AbortQueuedAfterErrorJob::Child_2' => :completed,
|
551
|
+
'AbortQueuedAfterErrorJob::Child_3' => :queued,
|
552
|
+
'AbortQueuedAfterErrorJob::Child_4' => :queued
|
516
553
|
|
517
554
|
execute queue
|
518
555
|
|
519
|
-
process.full_status.must_equal '
|
520
|
-
'
|
556
|
+
process.full_status.must_equal 'AbortQueuedAfterErrorJob' => :aborted,
|
557
|
+
'AbortQueuedAfterErrorJob::Child_1' => :waiting,
|
521
558
|
'Child_1_1' => :queued,
|
522
559
|
'Child_1_2' => :queued,
|
523
|
-
'
|
524
|
-
'
|
525
|
-
'
|
560
|
+
'AbortQueuedAfterErrorJob::Child_2' => :completed,
|
561
|
+
'AbortQueuedAfterErrorJob::Child_3' => :aborted,
|
562
|
+
'AbortQueuedAfterErrorJob::Child_4' => :queued
|
526
563
|
|
527
564
|
execute queue
|
528
565
|
|
529
|
-
process.full_status.must_equal '
|
530
|
-
'
|
566
|
+
process.full_status.must_equal 'AbortQueuedAfterErrorJob' => :aborted,
|
567
|
+
'AbortQueuedAfterErrorJob::Child_1' => :waiting,
|
531
568
|
'Child_1_1' => :queued,
|
532
569
|
'Child_1_2' => :queued,
|
533
|
-
'
|
534
|
-
'
|
535
|
-
'
|
570
|
+
'AbortQueuedAfterErrorJob::Child_2' => :completed,
|
571
|
+
'AbortQueuedAfterErrorJob::Child_3' => :aborted,
|
572
|
+
'AbortQueuedAfterErrorJob::Child_4' => :aborted
|
536
573
|
|
537
574
|
execute queue
|
538
575
|
|
539
|
-
process.full_status.must_equal '
|
540
|
-
'
|
576
|
+
process.full_status.must_equal 'AbortQueuedAfterErrorJob' => :aborted,
|
577
|
+
'AbortQueuedAfterErrorJob::Child_1' => :aborted,
|
541
578
|
'Child_1_1' => :aborted,
|
542
579
|
'Child_1_2' => :queued,
|
543
|
-
'
|
544
|
-
'
|
545
|
-
'
|
580
|
+
'AbortQueuedAfterErrorJob::Child_2' => :completed,
|
581
|
+
'AbortQueuedAfterErrorJob::Child_3' => :aborted,
|
582
|
+
'AbortQueuedAfterErrorJob::Child_4' => :aborted
|
546
583
|
|
547
584
|
execute queue
|
548
585
|
|
549
|
-
process.full_status.must_equal '
|
550
|
-
'
|
586
|
+
process.full_status.must_equal 'AbortQueuedAfterErrorJob' => :aborted,
|
587
|
+
'AbortQueuedAfterErrorJob::Child_1' => :aborted,
|
551
588
|
'Child_1_1' => :aborted,
|
552
589
|
'Child_1_2' => :aborted,
|
553
|
-
'
|
554
|
-
'
|
555
|
-
'
|
590
|
+
'AbortQueuedAfterErrorJob::Child_2' => :completed,
|
591
|
+
'AbortQueuedAfterErrorJob::Child_3' => :aborted,
|
592
|
+
'AbortQueuedAfterErrorJob::Child_4' => :aborted
|
556
593
|
|
557
594
|
process.real_error.must_equal 'Forced error'
|
595
|
+
process.processes[0].processes[1].error.message.must_equal Asynchronic::Process::AUTOMATIC_ABORTED_ERROR_MESSAGE
|
558
596
|
end
|
559
597
|
|
560
598
|
it 'Manual abort' do
|
@@ -617,36 +655,44 @@ module LifeCycleExamples
|
|
617
655
|
process_2.enqueue
|
618
656
|
execute queue
|
619
657
|
|
658
|
+
process_3 = create BasicJob
|
659
|
+
|
620
660
|
pid_1 = process_1.id
|
621
661
|
pid_2 = process_2.id
|
662
|
+
pid_3 = process_3.id
|
622
663
|
|
623
664
|
process_1.must_be_completed
|
624
665
|
process_2.must_be_waiting
|
666
|
+
process_3.must_be_pending
|
625
667
|
|
626
668
|
data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 53
|
627
669
|
data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 38
|
670
|
+
data_store.keys.select { |k| k.start_with? pid_3 }.count.must_equal 7
|
628
671
|
|
629
672
|
gc = Asynchronic::GarbageCollector.new env, 0.001
|
630
673
|
|
631
|
-
gc.add_condition('
|
674
|
+
gc.add_condition('Finalized', &:finalized?)
|
632
675
|
gc.add_condition('Waiting', &:waiting?)
|
633
676
|
gc.add_condition('Exception') { raise 'Invalid condition' }
|
634
677
|
|
635
|
-
gc.conditions_names.must_equal ['
|
678
|
+
gc.conditions_names.must_equal ['Finalized', 'Waiting', 'Exception']
|
636
679
|
|
637
680
|
gc.remove_condition 'Waiting'
|
638
681
|
|
639
|
-
gc.conditions_names.must_equal ['
|
682
|
+
gc.conditions_names.must_equal ['Finalized', 'Exception']
|
640
683
|
|
641
684
|
Thread.new do
|
642
685
|
sleep 0.01
|
643
686
|
gc.stop
|
644
687
|
end
|
645
688
|
|
646
|
-
|
689
|
+
Asynchronic::Process.stub_any_instance(:dead?, -> { id == pid_3 }) do
|
690
|
+
gc.start
|
691
|
+
end
|
647
692
|
|
648
693
|
data_store.keys.select { |k| k.start_with? pid_1 }.count.must_equal 0
|
649
694
|
data_store.keys.select { |k| k.start_with? pid_2 }.count.must_equal 38
|
695
|
+
data_store.keys.select { |k| k.start_with? pid_3 }.count.must_equal 0
|
650
696
|
end
|
651
697
|
|
652
698
|
it 'Before finalize hook when completed' do
|
@@ -717,5 +763,4 @@ module LifeCycleExamples
|
|
717
763
|
queue.must_be_empty
|
718
764
|
end
|
719
765
|
|
720
|
-
|
721
766
|
end
|
@@ -5,6 +5,7 @@ describe Asynchronic::Process, 'Life cycle - InMemory' do
|
|
5
5
|
|
6
6
|
let(:queue_engine) { Asynchronic::QueueEngine::InMemory.new }
|
7
7
|
let(:data_store) { Asynchronic::DataStore::InMemory.new }
|
8
|
+
let(:notifier) { Asynchronic::Notifier::InMemory.new }
|
8
9
|
|
9
10
|
include LifeCycleExamples
|
10
11
|
|
@@ -5,6 +5,7 @@ describe Asynchronic::Process, 'Life cycle - Redis' do
|
|
5
5
|
|
6
6
|
let(:queue_engine) { Asynchronic::QueueEngine::Ost.new }
|
7
7
|
let(:data_store) { Asynchronic::DataStore::Redis.new :asynchronic_test }
|
8
|
+
let(:notifier) { Asynchronic::Notifier::Broadcaster.new }
|
8
9
|
|
9
10
|
include LifeCycleExamples
|
10
11
|
|
data/spec/worker/redis_spec.rb
CHANGED
@@ -5,6 +5,7 @@ describe Asynchronic::Worker, 'Redis' do
|
|
5
5
|
|
6
6
|
let(:queue_engine) { Asynchronic::QueueEngine::Ost.new }
|
7
7
|
let(:data_store) { Asynchronic::DataStore::Redis.new :asynchronic_test}
|
8
|
+
let(:notifier) { Asynchronic::Notifier::Broadcaster.new }
|
8
9
|
|
9
10
|
after do
|
10
11
|
data_store.clear
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: asynchronic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gabriel Naiman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ost
|
@@ -25,21 +25,27 @@ dependencies:
|
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: broadcaster
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '1.0'
|
34
|
+
- - ">="
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: 1.0.2
|
34
37
|
type: :runtime
|
35
38
|
prerelease: false
|
36
39
|
version_requirements: !ruby/object:Gem::Requirement
|
37
40
|
requirements:
|
38
41
|
- - "~>"
|
39
42
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
43
|
+
version: '1.0'
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 1.0.2
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
48
|
+
name: class_config
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
51
|
- - "~>"
|
@@ -53,47 +59,47 @@ dependencies:
|
|
53
59
|
- !ruby/object:Gem::Version
|
54
60
|
version: '0.0'
|
55
61
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
62
|
+
name: transparent_proxy
|
57
63
|
requirement: !ruby/object:Gem::Requirement
|
58
64
|
requirements:
|
59
65
|
- - "~>"
|
60
66
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
67
|
+
version: '0.0'
|
62
68
|
type: :runtime
|
63
69
|
prerelease: false
|
64
70
|
version_requirements: !ruby/object:Gem::Requirement
|
65
71
|
requirements:
|
66
72
|
- - "~>"
|
67
73
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
74
|
+
version: '0.0'
|
69
75
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
76
|
+
name: multi_require
|
71
77
|
requirement: !ruby/object:Gem::Requirement
|
72
78
|
requirements:
|
73
79
|
- - "~>"
|
74
80
|
- !ruby/object:Gem::Version
|
75
|
-
version: '1.
|
76
|
-
type: :
|
81
|
+
version: '1.0'
|
82
|
+
type: :runtime
|
77
83
|
prerelease: false
|
78
84
|
version_requirements: !ruby/object:Gem::Requirement
|
79
85
|
requirements:
|
80
86
|
- - "~>"
|
81
87
|
- !ruby/object:Gem::Version
|
82
|
-
version: '1.
|
88
|
+
version: '1.0'
|
83
89
|
- !ruby/object:Gem::Dependency
|
84
90
|
name: rake
|
85
91
|
requirement: !ruby/object:Gem::Requirement
|
86
92
|
requirements:
|
87
93
|
- - "~>"
|
88
94
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
95
|
+
version: '12.0'
|
90
96
|
type: :development
|
91
97
|
prerelease: false
|
92
98
|
version_requirements: !ruby/object:Gem::Requirement
|
93
99
|
requirements:
|
94
100
|
- - "~>"
|
95
101
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
102
|
+
version: '12.0'
|
97
103
|
- !ruby/object:Gem::Dependency
|
98
104
|
name: minitest
|
99
105
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,6 +134,20 @@ dependencies:
|
|
128
134
|
- - "~>"
|
129
135
|
- !ruby/object:Gem::Version
|
130
136
|
version: '0.0'
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: minitest-stub_any_instance
|
139
|
+
requirement: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - "~>"
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: '1.0'
|
144
|
+
type: :development
|
145
|
+
prerelease: false
|
146
|
+
version_requirements: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - "~>"
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: '1.0'
|
131
151
|
- !ruby/object:Gem::Dependency
|
132
152
|
name: minitest-colorin
|
133
153
|
requirement: !ruby/object:Gem::Requirement
|
@@ -229,6 +249,8 @@ files:
|
|
229
249
|
- lib/asynchronic/error.rb
|
230
250
|
- lib/asynchronic/garbage_collector.rb
|
231
251
|
- lib/asynchronic/job.rb
|
252
|
+
- lib/asynchronic/notifier/broadcaster.rb
|
253
|
+
- lib/asynchronic/notifier/in_memory.rb
|
232
254
|
- lib/asynchronic/process.rb
|
233
255
|
- lib/asynchronic/queue_engine/in_memory.rb
|
234
256
|
- lib/asynchronic/queue_engine/ost.rb
|
@@ -275,8 +297,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
275
297
|
- !ruby/object:Gem::Version
|
276
298
|
version: '0'
|
277
299
|
requirements: []
|
278
|
-
|
279
|
-
rubygems_version: 2.6.8
|
300
|
+
rubygems_version: 3.0.6
|
280
301
|
signing_key:
|
281
302
|
specification_version: 4
|
282
303
|
summary: DSL for asynchronic pipeline using queues over Redis
|