asynchronic 3.0.3 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/README.md +1 -1
- data/lib/asynchronic.rb +8 -0
- data/lib/asynchronic/data_store/in_memory.rb +17 -15
- data/lib/asynchronic/data_store/key.rb +3 -3
- data/lib/asynchronic/data_store/lazy_value.rb +5 -3
- data/lib/asynchronic/data_store/redis.rb +22 -14
- data/lib/asynchronic/data_store/scoped_store.rb +18 -19
- data/lib/asynchronic/environment.rb +3 -3
- data/lib/asynchronic/error.rb +2 -3
- data/lib/asynchronic/job.rb +12 -12
- data/lib/asynchronic/notifier/broadcaster.rb +8 -4
- data/lib/asynchronic/process.rb +35 -32
- data/lib/asynchronic/queue_engine/in_memory.rb +17 -11
- data/lib/asynchronic/queue_engine/ost.rb +7 -5
- data/lib/asynchronic/queue_engine/synchronic.rb +19 -7
- data/lib/asynchronic/version.rb +1 -1
- data/lib/asynchronic/worker.rb +7 -10
- data/spec/data_store/data_store_examples.rb +17 -9
- data/spec/data_store/in_memory_spec.rb +0 -2
- data/spec/data_store/key_spec.rb +1 -1
- data/spec/data_store/lazy_value_examples.rb +7 -5
- data/spec/data_store/redis_spec.rb +4 -10
- data/spec/expectations.rb +2 -2
- data/spec/facade_spec.rb +3 -3
- data/spec/jobs.rb +10 -10
- data/spec/minitest_helper.rb +5 -12
- data/spec/process/life_cycle_examples.rb +24 -22
- data/spec/process/life_cycle_in_memory_spec.rb +0 -1
- data/spec/process/life_cycle_redis_spec.rb +0 -1
- data/spec/queue_engine/in_memory_spec.rb +1 -3
- data/spec/queue_engine/ost_spec.rb +1 -7
- data/spec/queue_engine/queue_engine_examples.rb +17 -9
- data/spec/queue_engine/synchronic_spec.rb +1 -1
- data/spec/worker/in_memory_spec.rb +1 -2
- data/spec/worker/redis_spec.rb +0 -6
- data/spec/worker/worker_examples.rb +6 -4
- metadata +3 -3
@@ -2,27 +2,25 @@ module Asynchronic
|
|
2
2
|
module QueueEngine
|
3
3
|
class InMemory
|
4
4
|
|
5
|
-
attr_reader :default_queue
|
6
|
-
|
7
5
|
def initialize(options={})
|
8
|
-
@
|
6
|
+
@options = options
|
9
7
|
@queues ||= Hash.new { |h,k| h[k] = Queue.new }
|
10
8
|
end
|
11
9
|
|
12
10
|
def default_queue
|
13
|
-
@default_queue ||= Asynchronic.default_queue
|
11
|
+
@default_queue ||= options.fetch(:default_queue, Asynchronic.default_queue)
|
14
12
|
end
|
15
13
|
|
16
14
|
def [](name)
|
17
|
-
|
15
|
+
queues[name]
|
18
16
|
end
|
19
17
|
|
20
|
-
def
|
21
|
-
|
18
|
+
def queue_names
|
19
|
+
queues.keys.map(&:to_sym)
|
22
20
|
end
|
23
21
|
|
24
22
|
def clear
|
25
|
-
|
23
|
+
queues.clear
|
26
24
|
end
|
27
25
|
|
28
26
|
def listener
|
@@ -37,12 +35,16 @@ module Asynchronic
|
|
37
35
|
[Asynchronic.connection_name]
|
38
36
|
end
|
39
37
|
|
38
|
+
private
|
39
|
+
|
40
|
+
attr_reader :queues, :options
|
41
|
+
|
40
42
|
|
41
43
|
class Queue
|
42
44
|
|
43
45
|
extend Forwardable
|
44
46
|
|
45
|
-
def_delegators
|
47
|
+
def_delegators :queue, :size, :empty?, :to_a
|
46
48
|
|
47
49
|
def initialize
|
48
50
|
@queue = []
|
@@ -50,13 +52,17 @@ module Asynchronic
|
|
50
52
|
end
|
51
53
|
|
52
54
|
def pop
|
53
|
-
|
55
|
+
mutex.synchronize { queue.shift }
|
54
56
|
end
|
55
57
|
|
56
58
|
def push(message)
|
57
|
-
|
59
|
+
mutex.synchronize { queue.push message }
|
58
60
|
end
|
59
61
|
|
62
|
+
private
|
63
|
+
|
64
|
+
attr_reader :queue, :mutex
|
65
|
+
|
60
66
|
end
|
61
67
|
|
62
68
|
|
@@ -5,22 +5,22 @@ module Asynchronic
|
|
5
5
|
attr_reader :redis, :default_queue
|
6
6
|
|
7
7
|
def initialize(options={})
|
8
|
-
@redis =
|
8
|
+
@redis = Asynchronic.establish_redis_connection options
|
9
9
|
@default_queue = options.fetch(:default_queue, Asynchronic.default_queue)
|
10
10
|
@queues ||= Hash.new { |h,k| h[k] = Queue.new k, redis }
|
11
11
|
@keep_alive_thread = notify_keep_alive
|
12
12
|
end
|
13
13
|
|
14
14
|
def [](name)
|
15
|
-
|
15
|
+
queues[name]
|
16
16
|
end
|
17
17
|
|
18
|
-
def
|
19
|
-
(
|
18
|
+
def queue_names
|
19
|
+
(queues.values.map(&:key) | redis.call!('KEYS', 'ost:*')).map { |q| q.to_s[4..-1].to_sym }
|
20
20
|
end
|
21
21
|
|
22
22
|
def clear
|
23
|
-
|
23
|
+
queues.clear
|
24
24
|
redis.call!('KEYS', 'ost:*').each { |k| redis.call!('DEL', k) }
|
25
25
|
end
|
26
26
|
|
@@ -41,6 +41,8 @@ module Asynchronic
|
|
41
41
|
|
42
42
|
private
|
43
43
|
|
44
|
+
attr_reader :queues
|
45
|
+
|
44
46
|
def notify_keep_alive
|
45
47
|
Thread.new do
|
46
48
|
loop do
|
@@ -5,7 +5,7 @@ module Asynchronic
|
|
5
5
|
attr_reader :stubs
|
6
6
|
|
7
7
|
def initialize(options={})
|
8
|
-
@
|
8
|
+
@options = options
|
9
9
|
@stubs = {}
|
10
10
|
end
|
11
11
|
|
@@ -14,7 +14,7 @@ module Asynchronic
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def environment
|
17
|
-
@environment ||= Asynchronic.environment
|
17
|
+
@environment ||= options.fetch(:environment, Asynchronic.environment)
|
18
18
|
end
|
19
19
|
|
20
20
|
def [](name)
|
@@ -22,7 +22,7 @@ module Asynchronic
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def stub(job, &block)
|
25
|
-
|
25
|
+
stubs[job] = block
|
26
26
|
end
|
27
27
|
|
28
28
|
def asynchronic?
|
@@ -33,6 +33,10 @@ module Asynchronic
|
|
33
33
|
[Asynchronic.connection_name]
|
34
34
|
end
|
35
35
|
|
36
|
+
private
|
37
|
+
|
38
|
+
attr_reader :options
|
39
|
+
|
36
40
|
|
37
41
|
class Queue
|
38
42
|
|
@@ -41,11 +45,11 @@ module Asynchronic
|
|
41
45
|
end
|
42
46
|
|
43
47
|
def push(message)
|
44
|
-
process =
|
48
|
+
process = engine.environment.load_process(message)
|
45
49
|
|
46
|
-
if
|
50
|
+
if engine.stubs[process.type]
|
47
51
|
job = process.job
|
48
|
-
block =
|
52
|
+
block = engine.stubs[process.type]
|
49
53
|
process.define_singleton_method :job do
|
50
54
|
MockJob.new job, process, &block
|
51
55
|
end
|
@@ -54,6 +58,10 @@ module Asynchronic
|
|
54
58
|
process.execute
|
55
59
|
end
|
56
60
|
|
61
|
+
private
|
62
|
+
|
63
|
+
attr_reader :engine
|
64
|
+
|
57
65
|
end
|
58
66
|
|
59
67
|
|
@@ -66,12 +74,16 @@ module Asynchronic
|
|
66
74
|
end
|
67
75
|
|
68
76
|
def call
|
69
|
-
|
77
|
+
block.call process
|
70
78
|
end
|
71
79
|
|
72
80
|
def before_finalize
|
73
81
|
end
|
74
82
|
|
83
|
+
private
|
84
|
+
|
85
|
+
attr_reader :process, :block
|
86
|
+
|
75
87
|
end
|
76
88
|
|
77
89
|
end
|
data/lib/asynchronic/version.rb
CHANGED
data/lib/asynchronic/worker.rb
CHANGED
@@ -1,24 +1,21 @@
|
|
1
1
|
class Asynchronic::Worker
|
2
2
|
|
3
|
-
attr_reader :queue
|
4
|
-
attr_reader :queue_name
|
5
|
-
attr_reader :env
|
6
|
-
attr_reader :listener
|
3
|
+
attr_reader :queue, :queue_name, :environment, :listener
|
7
4
|
|
8
|
-
def initialize(queue_name,
|
5
|
+
def initialize(queue_name, environment)
|
9
6
|
@queue_name = queue_name
|
10
|
-
@queue =
|
11
|
-
@
|
12
|
-
@listener =
|
7
|
+
@queue = environment.queue_engine[queue_name]
|
8
|
+
@environment = environment
|
9
|
+
@listener = environment.queue_engine.listener
|
13
10
|
end
|
14
11
|
|
15
12
|
def start
|
16
13
|
Asynchronic.logger.info('Asynchronic') { "Starting worker of #{queue_name} (#{Process.pid})" }
|
17
14
|
|
18
15
|
Signal.trap('QUIT') { stop }
|
19
|
-
|
16
|
+
|
20
17
|
listener.listen(queue) do |pid|
|
21
|
-
|
18
|
+
environment.load_process(pid).execute
|
22
19
|
end
|
23
20
|
end
|
24
21
|
|
@@ -1,5 +1,11 @@
|
|
1
1
|
module DataStoreExamples
|
2
|
-
|
2
|
+
|
3
|
+
extend Minitest::Spec::DSL
|
4
|
+
|
5
|
+
after do
|
6
|
+
data_store.clear
|
7
|
+
end
|
8
|
+
|
3
9
|
it 'Get/Set value' do
|
4
10
|
data_store[:key] = 123
|
5
11
|
data_store[:key].must_equal 123
|
@@ -32,8 +38,8 @@ module DataStoreExamples
|
|
32
38
|
data_store.delete_cascade Asynchronic::DataStore::Key[:key_1]
|
33
39
|
|
34
40
|
data_store.keys.sort.must_equal [
|
35
|
-
Asynchronic::DataStore::Key[:key_2],
|
36
|
-
Asynchronic::DataStore::Key[:key_2][:key_2_1],
|
41
|
+
Asynchronic::DataStore::Key[:key_2],
|
42
|
+
Asynchronic::DataStore::Key[:key_2][:key_2_1],
|
37
43
|
Asynchronic::DataStore::Key[:key_2][:key_2_2]
|
38
44
|
]
|
39
45
|
end
|
@@ -110,18 +116,20 @@ module DataStoreExamples
|
|
110
116
|
|
111
117
|
it 'Synchronization' do
|
112
118
|
sum = 0
|
113
|
-
|
119
|
+
|
120
|
+
threads = 10.times.map do
|
114
121
|
Thread.new do
|
115
|
-
data_store.synchronize('
|
122
|
+
data_store.synchronize('lock_key') do
|
116
123
|
temp = sum
|
117
|
-
sleep 0
|
124
|
+
sleep 0.01
|
118
125
|
sum = temp + 1
|
119
126
|
end
|
120
127
|
end
|
121
128
|
end
|
129
|
+
|
122
130
|
threads.each(&:join)
|
123
|
-
|
124
|
-
sum.must_equal
|
131
|
+
|
132
|
+
sum.must_equal threads.count
|
125
133
|
end
|
126
|
-
|
134
|
+
|
127
135
|
end
|
data/spec/data_store/key_spec.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
module LazyValueExamples
|
2
2
|
|
3
|
+
extend Minitest::Spec::DSL
|
4
|
+
|
3
5
|
def lazy_value(key)
|
4
6
|
Asynchronic::DataStore::LazyValue.new data_store, key
|
5
7
|
end
|
@@ -7,25 +9,25 @@ module LazyValueExamples
|
|
7
9
|
it 'Get' do
|
8
10
|
value = lazy_value :key
|
9
11
|
value.must_be_nil
|
10
|
-
|
11
|
-
data_store[:key] =
|
12
|
+
|
13
|
+
data_store[:key] = 1
|
12
14
|
value.must_equal 1
|
13
15
|
end
|
14
16
|
|
15
17
|
it 'Reload' do
|
16
18
|
value = lazy_value :key
|
17
19
|
|
18
|
-
data_store[:key] =
|
20
|
+
data_store[:key] = 1
|
19
21
|
value.must_equal 1
|
20
22
|
|
21
|
-
data_store[:key] =
|
23
|
+
data_store[:key] = 2
|
22
24
|
value.must_equal 1
|
23
25
|
value.reload.must_equal 2
|
24
26
|
end
|
25
27
|
|
26
28
|
it 'Transparent proxy' do
|
27
29
|
value = lazy_value :key
|
28
|
-
data_store[:key] =
|
30
|
+
data_store[:key] = 1
|
29
31
|
value.must_equal 1
|
30
32
|
end
|
31
33
|
|
@@ -1,17 +1,15 @@
|
|
1
1
|
require 'minitest_helper'
|
2
|
-
require_relative './data_store_examples'
|
3
|
-
require_relative './lazy_value_examples'
|
4
2
|
|
5
3
|
describe Asynchronic::DataStore::Redis do
|
6
4
|
|
7
5
|
let(:data_store) { Asynchronic::DataStore::Redis.new :asynchronic_test }
|
8
6
|
|
9
|
-
after do
|
10
|
-
data_store.clear
|
11
|
-
end
|
12
|
-
|
13
7
|
include DataStoreExamples
|
14
8
|
|
9
|
+
describe 'LazyValue' do
|
10
|
+
include LazyValueExamples
|
11
|
+
end
|
12
|
+
|
15
13
|
it 'Safe deserialization' do
|
16
14
|
SampleClass = Class.new
|
17
15
|
|
@@ -24,8 +22,4 @@ describe Asynchronic::DataStore::Redis do
|
|
24
22
|
data_store[:instance].must_be_instance_of String
|
25
23
|
end
|
26
24
|
|
27
|
-
describe 'LazyValue' do
|
28
|
-
include LazyValueExamples
|
29
|
-
end
|
30
|
-
|
31
25
|
end
|
data/spec/expectations.rb
CHANGED
@@ -15,10 +15,10 @@ module MiniTest::Assertions
|
|
15
15
|
def assert_be_initialized(process)
|
16
16
|
process.must_be :pending?
|
17
17
|
process.wont_be :finalized?
|
18
|
-
|
18
|
+
|
19
19
|
process.processes.must_be_empty
|
20
20
|
process.error.must_be_nil
|
21
|
-
|
21
|
+
|
22
22
|
process.created_at.must_be_instance_of Time
|
23
23
|
process.queued_at.must_be_nil
|
24
24
|
process.started_at.must_be_nil
|
data/spec/facade_spec.rb
CHANGED
@@ -6,7 +6,7 @@ describe Asynchronic, 'Facade' do
|
|
6
6
|
Asynchronic.environment.data_store.clear
|
7
7
|
Asynchronic.environment.queue_engine.clear
|
8
8
|
end
|
9
|
-
|
9
|
+
|
10
10
|
it 'Default queue' do
|
11
11
|
Asynchronic.default_queue.must_equal :asynchronic_test
|
12
12
|
end
|
@@ -40,7 +40,7 @@ describe Asynchronic, 'Facade' do
|
|
40
40
|
end
|
41
41
|
|
42
42
|
it 'List processes' do
|
43
|
-
ids = 3.times.map do
|
43
|
+
ids = 3.times.map do
|
44
44
|
process = Asynchronic.environment.create_process SequentialJob
|
45
45
|
process.id
|
46
46
|
end
|
@@ -51,7 +51,7 @@ describe Asynchronic, 'Facade' do
|
|
51
51
|
|
52
52
|
it 'Enqueue' do
|
53
53
|
id = BasicJob.enqueue input: 100
|
54
|
-
|
54
|
+
|
55
55
|
Asynchronic.environment.tap do |env|
|
56
56
|
process = env.load_process id
|
57
57
|
process.type.must_equal BasicJob
|
data/spec/jobs.rb
CHANGED
@@ -9,8 +9,8 @@ class SequentialJob < Asynchronic::Job
|
|
9
9
|
|
10
10
|
def call
|
11
11
|
async Step1, input: params[:input]
|
12
|
-
|
13
|
-
async Step2, dependency: Step1,
|
12
|
+
|
13
|
+
async Step2, dependency: Step1,
|
14
14
|
input: params[:input]
|
15
15
|
|
16
16
|
nil
|
@@ -32,7 +32,7 @@ end
|
|
32
32
|
|
33
33
|
|
34
34
|
class GraphJob < Asynchronic::Job
|
35
|
-
|
35
|
+
|
36
36
|
def call
|
37
37
|
async Sum, input: params[:input]
|
38
38
|
|
@@ -113,13 +113,13 @@ class AliasJob < Asynchronic::Job
|
|
113
113
|
def call
|
114
114
|
async Write, alias: :word_1,
|
115
115
|
text: 'Take'
|
116
|
-
|
117
|
-
async Write, alias: :word_2,
|
118
|
-
text: 'it',
|
116
|
+
|
117
|
+
async Write, alias: :word_2,
|
118
|
+
text: 'it',
|
119
119
|
prefix: result(:word_1)
|
120
|
-
|
121
|
-
async Write, alias: :word_3,
|
122
|
-
text: 'easy',
|
120
|
+
|
121
|
+
async Write, alias: :word_3,
|
122
|
+
text: 'easy',
|
123
123
|
prefix: result(:word_2)
|
124
124
|
|
125
125
|
result :word_3
|
@@ -402,4 +402,4 @@ class BeforeFinalizeExceptionOnAbortedJob < Asynchronic::Job
|
|
402
402
|
def before_finalize
|
403
403
|
raise 'Before finalize error'
|
404
404
|
end
|
405
|
-
end
|
405
|
+
end
|