asynchronic 3.0.3 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/README.md +1 -1
- data/lib/asynchronic.rb +8 -0
- data/lib/asynchronic/data_store/in_memory.rb +17 -15
- data/lib/asynchronic/data_store/key.rb +3 -3
- data/lib/asynchronic/data_store/lazy_value.rb +5 -3
- data/lib/asynchronic/data_store/redis.rb +22 -14
- data/lib/asynchronic/data_store/scoped_store.rb +18 -19
- data/lib/asynchronic/environment.rb +3 -3
- data/lib/asynchronic/error.rb +2 -3
- data/lib/asynchronic/job.rb +12 -12
- data/lib/asynchronic/notifier/broadcaster.rb +8 -4
- data/lib/asynchronic/process.rb +35 -32
- data/lib/asynchronic/queue_engine/in_memory.rb +17 -11
- data/lib/asynchronic/queue_engine/ost.rb +7 -5
- data/lib/asynchronic/queue_engine/synchronic.rb +19 -7
- data/lib/asynchronic/version.rb +1 -1
- data/lib/asynchronic/worker.rb +7 -10
- data/spec/data_store/data_store_examples.rb +17 -9
- data/spec/data_store/in_memory_spec.rb +0 -2
- data/spec/data_store/key_spec.rb +1 -1
- data/spec/data_store/lazy_value_examples.rb +7 -5
- data/spec/data_store/redis_spec.rb +4 -10
- data/spec/expectations.rb +2 -2
- data/spec/facade_spec.rb +3 -3
- data/spec/jobs.rb +10 -10
- data/spec/minitest_helper.rb +5 -12
- data/spec/process/life_cycle_examples.rb +24 -22
- data/spec/process/life_cycle_in_memory_spec.rb +0 -1
- data/spec/process/life_cycle_redis_spec.rb +0 -1
- data/spec/queue_engine/in_memory_spec.rb +1 -3
- data/spec/queue_engine/ost_spec.rb +1 -7
- data/spec/queue_engine/queue_engine_examples.rb +17 -9
- data/spec/queue_engine/synchronic_spec.rb +1 -1
- data/spec/worker/in_memory_spec.rb +1 -2
- data/spec/worker/redis_spec.rb +0 -6
- data/spec/worker/worker_examples.rb +6 -4
- metadata +3 -3
@@ -2,27 +2,25 @@ module Asynchronic
|
|
2
2
|
module QueueEngine
|
3
3
|
class InMemory
|
4
4
|
|
5
|
-
attr_reader :default_queue
|
6
|
-
|
7
5
|
def initialize(options={})
|
8
|
-
@
|
6
|
+
@options = options
|
9
7
|
@queues ||= Hash.new { |h,k| h[k] = Queue.new }
|
10
8
|
end
|
11
9
|
|
12
10
|
def default_queue
|
13
|
-
@default_queue ||= Asynchronic.default_queue
|
11
|
+
@default_queue ||= options.fetch(:default_queue, Asynchronic.default_queue)
|
14
12
|
end
|
15
13
|
|
16
14
|
def [](name)
|
17
|
-
|
15
|
+
queues[name]
|
18
16
|
end
|
19
17
|
|
20
|
-
def
|
21
|
-
|
18
|
+
def queue_names
|
19
|
+
queues.keys.map(&:to_sym)
|
22
20
|
end
|
23
21
|
|
24
22
|
def clear
|
25
|
-
|
23
|
+
queues.clear
|
26
24
|
end
|
27
25
|
|
28
26
|
def listener
|
@@ -37,12 +35,16 @@ module Asynchronic
|
|
37
35
|
[Asynchronic.connection_name]
|
38
36
|
end
|
39
37
|
|
38
|
+
private
|
39
|
+
|
40
|
+
attr_reader :queues, :options
|
41
|
+
|
40
42
|
|
41
43
|
class Queue
|
42
44
|
|
43
45
|
extend Forwardable
|
44
46
|
|
45
|
-
def_delegators
|
47
|
+
def_delegators :queue, :size, :empty?, :to_a
|
46
48
|
|
47
49
|
def initialize
|
48
50
|
@queue = []
|
@@ -50,13 +52,17 @@ module Asynchronic
|
|
50
52
|
end
|
51
53
|
|
52
54
|
def pop
|
53
|
-
|
55
|
+
mutex.synchronize { queue.shift }
|
54
56
|
end
|
55
57
|
|
56
58
|
def push(message)
|
57
|
-
|
59
|
+
mutex.synchronize { queue.push message }
|
58
60
|
end
|
59
61
|
|
62
|
+
private
|
63
|
+
|
64
|
+
attr_reader :queue, :mutex
|
65
|
+
|
60
66
|
end
|
61
67
|
|
62
68
|
|
@@ -5,22 +5,22 @@ module Asynchronic
|
|
5
5
|
attr_reader :redis, :default_queue
|
6
6
|
|
7
7
|
def initialize(options={})
|
8
|
-
@redis =
|
8
|
+
@redis = Asynchronic.establish_redis_connection options
|
9
9
|
@default_queue = options.fetch(:default_queue, Asynchronic.default_queue)
|
10
10
|
@queues ||= Hash.new { |h,k| h[k] = Queue.new k, redis }
|
11
11
|
@keep_alive_thread = notify_keep_alive
|
12
12
|
end
|
13
13
|
|
14
14
|
def [](name)
|
15
|
-
|
15
|
+
queues[name]
|
16
16
|
end
|
17
17
|
|
18
|
-
def
|
19
|
-
(
|
18
|
+
def queue_names
|
19
|
+
(queues.values.map(&:key) | redis.call!('KEYS', 'ost:*')).map { |q| q.to_s[4..-1].to_sym }
|
20
20
|
end
|
21
21
|
|
22
22
|
def clear
|
23
|
-
|
23
|
+
queues.clear
|
24
24
|
redis.call!('KEYS', 'ost:*').each { |k| redis.call!('DEL', k) }
|
25
25
|
end
|
26
26
|
|
@@ -41,6 +41,8 @@ module Asynchronic
|
|
41
41
|
|
42
42
|
private
|
43
43
|
|
44
|
+
attr_reader :queues
|
45
|
+
|
44
46
|
def notify_keep_alive
|
45
47
|
Thread.new do
|
46
48
|
loop do
|
@@ -5,7 +5,7 @@ module Asynchronic
|
|
5
5
|
attr_reader :stubs
|
6
6
|
|
7
7
|
def initialize(options={})
|
8
|
-
@
|
8
|
+
@options = options
|
9
9
|
@stubs = {}
|
10
10
|
end
|
11
11
|
|
@@ -14,7 +14,7 @@ module Asynchronic
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def environment
|
17
|
-
@environment ||= Asynchronic.environment
|
17
|
+
@environment ||= options.fetch(:environment, Asynchronic.environment)
|
18
18
|
end
|
19
19
|
|
20
20
|
def [](name)
|
@@ -22,7 +22,7 @@ module Asynchronic
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def stub(job, &block)
|
25
|
-
|
25
|
+
stubs[job] = block
|
26
26
|
end
|
27
27
|
|
28
28
|
def asynchronic?
|
@@ -33,6 +33,10 @@ module Asynchronic
|
|
33
33
|
[Asynchronic.connection_name]
|
34
34
|
end
|
35
35
|
|
36
|
+
private
|
37
|
+
|
38
|
+
attr_reader :options
|
39
|
+
|
36
40
|
|
37
41
|
class Queue
|
38
42
|
|
@@ -41,11 +45,11 @@ module Asynchronic
|
|
41
45
|
end
|
42
46
|
|
43
47
|
def push(message)
|
44
|
-
process =
|
48
|
+
process = engine.environment.load_process(message)
|
45
49
|
|
46
|
-
if
|
50
|
+
if engine.stubs[process.type]
|
47
51
|
job = process.job
|
48
|
-
block =
|
52
|
+
block = engine.stubs[process.type]
|
49
53
|
process.define_singleton_method :job do
|
50
54
|
MockJob.new job, process, &block
|
51
55
|
end
|
@@ -54,6 +58,10 @@ module Asynchronic
|
|
54
58
|
process.execute
|
55
59
|
end
|
56
60
|
|
61
|
+
private
|
62
|
+
|
63
|
+
attr_reader :engine
|
64
|
+
|
57
65
|
end
|
58
66
|
|
59
67
|
|
@@ -66,12 +74,16 @@ module Asynchronic
|
|
66
74
|
end
|
67
75
|
|
68
76
|
def call
|
69
|
-
|
77
|
+
block.call process
|
70
78
|
end
|
71
79
|
|
72
80
|
def before_finalize
|
73
81
|
end
|
74
82
|
|
83
|
+
private
|
84
|
+
|
85
|
+
attr_reader :process, :block
|
86
|
+
|
75
87
|
end
|
76
88
|
|
77
89
|
end
|
data/lib/asynchronic/version.rb
CHANGED
data/lib/asynchronic/worker.rb
CHANGED
@@ -1,24 +1,21 @@
|
|
1
1
|
class Asynchronic::Worker
|
2
2
|
|
3
|
-
attr_reader :queue
|
4
|
-
attr_reader :queue_name
|
5
|
-
attr_reader :env
|
6
|
-
attr_reader :listener
|
3
|
+
attr_reader :queue, :queue_name, :environment, :listener
|
7
4
|
|
8
|
-
def initialize(queue_name,
|
5
|
+
def initialize(queue_name, environment)
|
9
6
|
@queue_name = queue_name
|
10
|
-
@queue =
|
11
|
-
@
|
12
|
-
@listener =
|
7
|
+
@queue = environment.queue_engine[queue_name]
|
8
|
+
@environment = environment
|
9
|
+
@listener = environment.queue_engine.listener
|
13
10
|
end
|
14
11
|
|
15
12
|
def start
|
16
13
|
Asynchronic.logger.info('Asynchronic') { "Starting worker of #{queue_name} (#{Process.pid})" }
|
17
14
|
|
18
15
|
Signal.trap('QUIT') { stop }
|
19
|
-
|
16
|
+
|
20
17
|
listener.listen(queue) do |pid|
|
21
|
-
|
18
|
+
environment.load_process(pid).execute
|
22
19
|
end
|
23
20
|
end
|
24
21
|
|
@@ -1,5 +1,11 @@
|
|
1
1
|
module DataStoreExamples
|
2
|
-
|
2
|
+
|
3
|
+
extend Minitest::Spec::DSL
|
4
|
+
|
5
|
+
after do
|
6
|
+
data_store.clear
|
7
|
+
end
|
8
|
+
|
3
9
|
it 'Get/Set value' do
|
4
10
|
data_store[:key] = 123
|
5
11
|
data_store[:key].must_equal 123
|
@@ -32,8 +38,8 @@ module DataStoreExamples
|
|
32
38
|
data_store.delete_cascade Asynchronic::DataStore::Key[:key_1]
|
33
39
|
|
34
40
|
data_store.keys.sort.must_equal [
|
35
|
-
Asynchronic::DataStore::Key[:key_2],
|
36
|
-
Asynchronic::DataStore::Key[:key_2][:key_2_1],
|
41
|
+
Asynchronic::DataStore::Key[:key_2],
|
42
|
+
Asynchronic::DataStore::Key[:key_2][:key_2_1],
|
37
43
|
Asynchronic::DataStore::Key[:key_2][:key_2_2]
|
38
44
|
]
|
39
45
|
end
|
@@ -110,18 +116,20 @@ module DataStoreExamples
|
|
110
116
|
|
111
117
|
it 'Synchronization' do
|
112
118
|
sum = 0
|
113
|
-
|
119
|
+
|
120
|
+
threads = 10.times.map do
|
114
121
|
Thread.new do
|
115
|
-
data_store.synchronize('
|
122
|
+
data_store.synchronize('lock_key') do
|
116
123
|
temp = sum
|
117
|
-
sleep 0
|
124
|
+
sleep 0.01
|
118
125
|
sum = temp + 1
|
119
126
|
end
|
120
127
|
end
|
121
128
|
end
|
129
|
+
|
122
130
|
threads.each(&:join)
|
123
|
-
|
124
|
-
sum.must_equal
|
131
|
+
|
132
|
+
sum.must_equal threads.count
|
125
133
|
end
|
126
|
-
|
134
|
+
|
127
135
|
end
|
data/spec/data_store/key_spec.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
module LazyValueExamples
|
2
2
|
|
3
|
+
extend Minitest::Spec::DSL
|
4
|
+
|
3
5
|
def lazy_value(key)
|
4
6
|
Asynchronic::DataStore::LazyValue.new data_store, key
|
5
7
|
end
|
@@ -7,25 +9,25 @@ module LazyValueExamples
|
|
7
9
|
it 'Get' do
|
8
10
|
value = lazy_value :key
|
9
11
|
value.must_be_nil
|
10
|
-
|
11
|
-
data_store[:key] =
|
12
|
+
|
13
|
+
data_store[:key] = 1
|
12
14
|
value.must_equal 1
|
13
15
|
end
|
14
16
|
|
15
17
|
it 'Reload' do
|
16
18
|
value = lazy_value :key
|
17
19
|
|
18
|
-
data_store[:key] =
|
20
|
+
data_store[:key] = 1
|
19
21
|
value.must_equal 1
|
20
22
|
|
21
|
-
data_store[:key] =
|
23
|
+
data_store[:key] = 2
|
22
24
|
value.must_equal 1
|
23
25
|
value.reload.must_equal 2
|
24
26
|
end
|
25
27
|
|
26
28
|
it 'Transparent proxy' do
|
27
29
|
value = lazy_value :key
|
28
|
-
data_store[:key] =
|
30
|
+
data_store[:key] = 1
|
29
31
|
value.must_equal 1
|
30
32
|
end
|
31
33
|
|
@@ -1,17 +1,15 @@
|
|
1
1
|
require 'minitest_helper'
|
2
|
-
require_relative './data_store_examples'
|
3
|
-
require_relative './lazy_value_examples'
|
4
2
|
|
5
3
|
describe Asynchronic::DataStore::Redis do
|
6
4
|
|
7
5
|
let(:data_store) { Asynchronic::DataStore::Redis.new :asynchronic_test }
|
8
6
|
|
9
|
-
after do
|
10
|
-
data_store.clear
|
11
|
-
end
|
12
|
-
|
13
7
|
include DataStoreExamples
|
14
8
|
|
9
|
+
describe 'LazyValue' do
|
10
|
+
include LazyValueExamples
|
11
|
+
end
|
12
|
+
|
15
13
|
it 'Safe deserialization' do
|
16
14
|
SampleClass = Class.new
|
17
15
|
|
@@ -24,8 +22,4 @@ describe Asynchronic::DataStore::Redis do
|
|
24
22
|
data_store[:instance].must_be_instance_of String
|
25
23
|
end
|
26
24
|
|
27
|
-
describe 'LazyValue' do
|
28
|
-
include LazyValueExamples
|
29
|
-
end
|
30
|
-
|
31
25
|
end
|
data/spec/expectations.rb
CHANGED
@@ -15,10 +15,10 @@ module MiniTest::Assertions
|
|
15
15
|
def assert_be_initialized(process)
|
16
16
|
process.must_be :pending?
|
17
17
|
process.wont_be :finalized?
|
18
|
-
|
18
|
+
|
19
19
|
process.processes.must_be_empty
|
20
20
|
process.error.must_be_nil
|
21
|
-
|
21
|
+
|
22
22
|
process.created_at.must_be_instance_of Time
|
23
23
|
process.queued_at.must_be_nil
|
24
24
|
process.started_at.must_be_nil
|
data/spec/facade_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe Asynchronic, 'Facade' do
|
|
6
6
|
Asynchronic.environment.data_store.clear
|
7
7
|
Asynchronic.environment.queue_engine.clear
|
8
8
|
end
|
9
|
-
|
9
|
+
|
10
10
|
it 'Default queue' do
|
11
11
|
Asynchronic.default_queue.must_equal :asynchronic_test
|
12
12
|
end
|
@@ -40,7 +40,7 @@ describe Asynchronic, 'Facade' do
|
|
40
40
|
end
|
41
41
|
|
42
42
|
it 'List processes' do
|
43
|
-
ids = 3.times.map do
|
43
|
+
ids = 3.times.map do
|
44
44
|
process = Asynchronic.environment.create_process SequentialJob
|
45
45
|
process.id
|
46
46
|
end
|
@@ -51,7 +51,7 @@ describe Asynchronic, 'Facade' do
|
|
51
51
|
|
52
52
|
it 'Enqueue' do
|
53
53
|
id = BasicJob.enqueue input: 100
|
54
|
-
|
54
|
+
|
55
55
|
Asynchronic.environment.tap do |env|
|
56
56
|
process = env.load_process id
|
57
57
|
process.type.must_equal BasicJob
|
data/spec/jobs.rb
CHANGED
@@ -9,8 +9,8 @@ class SequentialJob < Asynchronic::Job
|
|
9
9
|
|
10
10
|
def call
|
11
11
|
async Step1, input: params[:input]
|
12
|
-
|
13
|
-
async Step2, dependency: Step1,
|
12
|
+
|
13
|
+
async Step2, dependency: Step1,
|
14
14
|
input: params[:input]
|
15
15
|
|
16
16
|
nil
|
@@ -32,7 +32,7 @@ end
|
|
32
32
|
|
33
33
|
|
34
34
|
class GraphJob < Asynchronic::Job
|
35
|
-
|
35
|
+
|
36
36
|
def call
|
37
37
|
async Sum, input: params[:input]
|
38
38
|
|
@@ -113,13 +113,13 @@ class AliasJob < Asynchronic::Job
|
|
113
113
|
def call
|
114
114
|
async Write, alias: :word_1,
|
115
115
|
text: 'Take'
|
116
|
-
|
117
|
-
async Write, alias: :word_2,
|
118
|
-
text: 'it',
|
116
|
+
|
117
|
+
async Write, alias: :word_2,
|
118
|
+
text: 'it',
|
119
119
|
prefix: result(:word_1)
|
120
|
-
|
121
|
-
async Write, alias: :word_3,
|
122
|
-
text: 'easy',
|
120
|
+
|
121
|
+
async Write, alias: :word_3,
|
122
|
+
text: 'easy',
|
123
123
|
prefix: result(:word_2)
|
124
124
|
|
125
125
|
result :word_3
|
@@ -402,4 +402,4 @@ class BeforeFinalizeExceptionOnAbortedJob < Asynchronic::Job
|
|
402
402
|
def before_finalize
|
403
403
|
raise 'Before finalize error'
|
404
404
|
end
|
405
|
-
end
|
405
|
+
end
|