asynchronic 3.0.3 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -0
  3. data/README.md +1 -1
  4. data/lib/asynchronic.rb +8 -0
  5. data/lib/asynchronic/data_store/in_memory.rb +17 -15
  6. data/lib/asynchronic/data_store/key.rb +3 -3
  7. data/lib/asynchronic/data_store/lazy_value.rb +5 -3
  8. data/lib/asynchronic/data_store/redis.rb +22 -14
  9. data/lib/asynchronic/data_store/scoped_store.rb +18 -19
  10. data/lib/asynchronic/environment.rb +3 -3
  11. data/lib/asynchronic/error.rb +2 -3
  12. data/lib/asynchronic/job.rb +12 -12
  13. data/lib/asynchronic/notifier/broadcaster.rb +8 -4
  14. data/lib/asynchronic/process.rb +35 -32
  15. data/lib/asynchronic/queue_engine/in_memory.rb +17 -11
  16. data/lib/asynchronic/queue_engine/ost.rb +7 -5
  17. data/lib/asynchronic/queue_engine/synchronic.rb +19 -7
  18. data/lib/asynchronic/version.rb +1 -1
  19. data/lib/asynchronic/worker.rb +7 -10
  20. data/spec/data_store/data_store_examples.rb +17 -9
  21. data/spec/data_store/in_memory_spec.rb +0 -2
  22. data/spec/data_store/key_spec.rb +1 -1
  23. data/spec/data_store/lazy_value_examples.rb +7 -5
  24. data/spec/data_store/redis_spec.rb +4 -10
  25. data/spec/expectations.rb +2 -2
  26. data/spec/facade_spec.rb +3 -3
  27. data/spec/jobs.rb +10 -10
  28. data/spec/minitest_helper.rb +5 -12
  29. data/spec/process/life_cycle_examples.rb +24 -22
  30. data/spec/process/life_cycle_in_memory_spec.rb +0 -1
  31. data/spec/process/life_cycle_redis_spec.rb +0 -1
  32. data/spec/queue_engine/in_memory_spec.rb +1 -3
  33. data/spec/queue_engine/ost_spec.rb +1 -7
  34. data/spec/queue_engine/queue_engine_examples.rb +17 -9
  35. data/spec/queue_engine/synchronic_spec.rb +1 -1
  36. data/spec/worker/in_memory_spec.rb +1 -2
  37. data/spec/worker/redis_spec.rb +0 -6
  38. data/spec/worker/worker_examples.rb +6 -4
  39. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7cc53216188aca01f66806e933e68d3edacda3dabca5e93b3da3eae1a41bbae5
4
- data.tar.gz: ad209146fc5b92f5d0d38c5d0ce27b72b6c6f7308831f9facb08a9bc7a538904
3
+ metadata.gz: 47872aa8b0fdee31f769749a1485e7be5ca2d5f9b7de51863c223547cad4849b
4
+ data.tar.gz: d7f0984211e6bfc46d1960a3b53aef3d3f244a619005504d7ba5e6c27612dbdd
5
5
  SHA512:
6
- metadata.gz: 3e826e042a938689168e9ca085d4fd5f28250ed350ebdb658c871415b36df88088e290712c72558256c1535b247ead901fbae3818e692895f6e4d8052ab5f560
7
- data.tar.gz: 2a1d517fb7cd29556d790e6680928e9294e79329b23a3e7ebab6e948ebc6a087d713cc2c3ca12908ca25f4798585198d35a88351c3f0723ad47d8e4b4416bb2d
6
+ metadata.gz: 1893074c7181e0eea27a276e781fdfc1680c909986ccbc80645aeafbd08ef44e3d23cfc82a3480bf1f453987aa75f0667aafd9b5c1f10ddfed5ec28067f2684b
7
+ data.tar.gz: f580a9b1660a3e1aa5fcac083b27cb73c169e2d68ac974114ddff6eb2cc24267deac81af780c4ccc2b84106021e6fbd554269674af9ec89bacd26f5f7c1714d0
data/.travis.yml CHANGED
@@ -9,6 +9,7 @@ rvm:
9
9
  - 2.5
10
10
  - 2.6
11
11
  - 2.7
12
+ - 3.0
12
13
  - jruby-9.2.9.0
13
14
  - ruby-head
14
15
  - jruby-head
data/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # Asynchronic
2
2
 
3
3
  [![Gem Version](https://badge.fury.io/rb/asynchronic.svg)](https://rubygems.org/gems/asynchronic)
4
- [![Build Status](https://travis-ci.org/gabynaiman/asynchronic.svg?branch=master)](https://travis-ci.org/gabynaiman/asynchronic)
4
+ [![Build Status](https://travis-ci.com/gabynaiman/asynchronic.svg?branch=master)](https://travis-ci.com/gabynaiman/asynchronic)
5
5
  [![Coverage Status](https://coveralls.io/repos/gabynaiman/asynchronic/badge.svg?branch=master)](https://coveralls.io/r/gabynaiman/asynchronic?branch=master)
6
6
  [![Code Climate](https://codeclimate.com/github/gabynaiman/asynchronic.svg)](https://codeclimate.com/github/gabynaiman/asynchronic)
7
7
 
data/lib/asynchronic.rb CHANGED
@@ -23,6 +23,8 @@ module Asynchronic
23
23
  attr_config :logger, Logger.new($stdout)
24
24
  attr_config :retry_timeout, 30
25
25
  attr_config :garbage_collector_timeout, 30
26
+ attr_config :redis_client, Redic
27
+ attr_config :redis_settings, 'redis://localhost:6379'
26
28
  attr_config :redis_data_store_sync_timeout, 0.01
27
29
  attr_config :keep_alive_timeout, 1
28
30
  attr_config :connection_name, "HOST=#{Socket.gethostname},PID=#{::Process.pid},UUID=#{SecureRandom.uuid}"
@@ -43,6 +45,12 @@ module Asynchronic
43
45
  @garbage_collector ||= GarbageCollector.new environment, garbage_collector_timeout
44
46
  end
45
47
 
48
+ def self.establish_redis_connection(options={})
49
+ redis_client = options.fetch(:redis_client, Asynchronic.redis_client)
50
+ redis_settings = options.fetch(:redis_settings, Asynchronic.redis_settings)
51
+ redis_client.new redis_settings
52
+ end
53
+
46
54
  def self.retry_execution(klass, message)
47
55
  begin
48
56
  result = yield
@@ -4,7 +4,15 @@ module Asynchronic
4
4
 
5
5
  include Helper
6
6
 
7
- def initialize(hash={})
7
+ def self.connect(object_id)
8
+ connections[object_id]
9
+ end
10
+
11
+ def self.connections
12
+ @connections ||= {}
13
+ end
14
+
15
+ def initialize
8
16
  @hash = {}
9
17
  @mutex = Mutex.new
10
18
  @keys_mutex = Hash.new { |h,k| h[k] = Mutex.new }
@@ -12,43 +20,37 @@ module Asynchronic
12
20
  end
13
21
 
14
22
  def [](key)
15
- Marshal.load(@hash[key.to_s]) if @hash.key? key.to_s
23
+ Marshal.load(hash[key.to_s]) if hash.key? key.to_s
16
24
  end
17
25
 
18
26
  def []=(key, value)
19
- @mutex.synchronize { @hash[key.to_s] = Marshal.dump(value) }
27
+ mutex.synchronize { hash[key.to_s] = Marshal.dump(value) }
20
28
  end
21
29
 
22
30
  def delete(key)
23
- @hash.delete key.to_s
31
+ hash.delete key.to_s
24
32
  end
25
33
 
26
34
  def delete_cascade(key)
27
- keys = self.keys.select { |k| k.sections.first == key }
28
- keys.each { |k| delete k }
35
+ keys.select { |k| k.sections.first == key }
36
+ .each { |k| delete k }
29
37
  end
30
38
 
31
39
  def keys
32
- @hash.keys.map { |k| Key[k] }
40
+ hash.keys.map { |k| Key[k] }
33
41
  end
34
42
 
35
43
  def synchronize(key, &block)
36
- @keys_mutex[key].synchronize(&block)
44
+ keys_mutex[key].synchronize(&block)
37
45
  end
38
46
 
39
47
  def connection_args
40
48
  [object_id]
41
49
  end
42
50
 
43
- def self.connect(object_id)
44
- connections[object_id]
45
- end
46
-
47
51
  private
48
52
 
49
- def self.connections
50
- @connections ||= {}
51
- end
53
+ attr_reader :hash, :mutex, :keys_mutex
52
54
 
53
55
  end
54
56
  end
@@ -1,19 +1,19 @@
1
1
  module Asynchronic
2
2
  module DataStore
3
3
  class Key < String
4
-
4
+
5
5
  SEPARATOR = '|'
6
6
 
7
7
  def self.[](key)
8
8
  new key
9
9
  end
10
-
10
+
11
11
  def initialize(key)
12
12
  super key.to_s
13
13
  end
14
14
 
15
15
  def [](key)
16
- self.class.new [self,key].join(SEPARATOR)
16
+ self.class.new [self, key].join(SEPARATOR)
17
17
  end
18
18
 
19
19
  def sections
@@ -14,11 +14,11 @@ module Asynchronic
14
14
  end
15
15
 
16
16
  def inspect
17
- "#<#{proxy_class} @data_store_class=#{@data_store_class} @data_store_connection_args=#{@data_store_connection_args} @key=#{@key}>"
17
+ "#<#{proxy_class} @data_store_class=#{data_store_class} @data_store_connection_args=#{data_store_connection_args} @key=#{key}>"
18
18
  end
19
19
 
20
20
  def data_store
21
- @data_store_class.connect(*@data_store_connection_args)
21
+ data_store_class.connect(*data_store_connection_args)
22
22
  end
23
23
 
24
24
  def to_value
@@ -27,8 +27,10 @@ module Asynchronic
27
27
 
28
28
  private
29
29
 
30
+ attr_reader :data_store_class, :data_store_connection_args, :key
31
+
30
32
  def __getobj__
31
- @value ||= data_store[@key]
33
+ @value ||= data_store[key]
32
34
  end
33
35
 
34
36
  end
@@ -6,13 +6,17 @@ module Asynchronic
6
6
 
7
7
  include Helper
8
8
 
9
- def initialize(scope, *args)
9
+ def self.connect(*args)
10
+ new(*args)
11
+ end
12
+
13
+ def initialize(scope, options={})
10
14
  @scope = Key[scope]
11
- @redis = Redic.new(*args)
15
+ @options = options
12
16
  end
13
17
 
14
18
  def [](key)
15
- value = @redis.call! 'GET', @scope[key]
19
+ value = redis.call! 'GET', scope[key]
16
20
  value ? Marshal.load(value) : nil
17
21
  rescue => ex
18
22
  Asynchronic.logger.warn('Asynchronic') { ex.message }
@@ -20,39 +24,43 @@ module Asynchronic
20
24
  end
21
25
 
22
26
  def []=(key, value)
23
- @redis.call! 'SET', @scope[key], Marshal.dump(value)
27
+ redis.call! 'SET', scope[key], Marshal.dump(value)
24
28
  end
25
29
 
26
30
  def delete(key)
27
- @redis.call! 'DEL', @scope[key]
31
+ redis.call! 'DEL', scope[key]
28
32
  end
29
33
 
30
34
  def delete_cascade(key)
31
- @redis.call! 'DEL', @scope[key]
32
- @redis.call!('KEYS', @scope[key]['*']).each { |k| @redis.call! 'DEL', k }
35
+ redis.call! 'DEL', scope[key]
36
+ redis.call!('KEYS', scope[key]['*']).each { |k| redis.call! 'DEL', k }
33
37
  end
34
38
 
35
39
  def keys
36
- @redis.call!('KEYS', @scope['*']).map { |k| Key[k].remove_first }
40
+ redis.call!('KEYS', scope['*']).map { |k| Key[k].remove_first }
37
41
  end
38
42
 
39
43
  def synchronize(key)
40
- while @redis.call!('GETSET', @scope[key][LOCKED], LOCKED) == LOCKED
44
+ while redis.call!('GETSET', scope[key][LOCKED], LOCKED) == LOCKED
41
45
  sleep Asynchronic.redis_data_store_sync_timeout
42
46
  end
43
47
  yield
44
48
  ensure
45
- @redis.call! 'DEL', @scope[key][LOCKED]
49
+ redis.call! 'DEL', scope[key][LOCKED]
46
50
  end
47
51
 
48
52
  def connection_args
49
- [@scope, @redis.url]
53
+ [scope, options]
50
54
  end
51
55
 
52
- def self.connect(*args)
53
- new(*args)
56
+ private
57
+
58
+ attr_reader :scope, :options
59
+
60
+ def redis
61
+ @redis ||= Asynchronic.establish_redis_connection options
54
62
  end
55
-
63
+
56
64
  end
57
65
  end
58
66
  end
@@ -4,34 +4,38 @@ module Asynchronic
4
4
 
5
5
  include Helper
6
6
 
7
- attr_reader :data_store
8
- attr_reader :scope
9
-
7
+ attr_reader :data_store, :scope
8
+
9
+ def self.connect(options)
10
+ data_store = options[:data_store_class].connect(*options[:data_store_connection_args])
11
+ new data_store, options[:scope]
12
+ end
13
+
10
14
  def initialize(data_store, scope)
11
15
  @data_store = data_store
12
16
  @scope = Key[scope]
13
17
  end
14
18
 
15
19
  def [](key)
16
- @data_store[@scope[key]]
20
+ data_store[scope[key]]
17
21
  end
18
22
 
19
23
  def []=(key, value)
20
- @data_store[@scope[key]] = value
24
+ data_store[scope[key]] = value
21
25
  end
22
26
 
23
27
  def delete(key)
24
- @data_store.delete @scope[key]
28
+ data_store.delete scope[key]
25
29
  end
26
30
 
27
31
  def delete_cascade
28
- @data_store.delete_cascade @scope
32
+ data_store.delete_cascade scope
29
33
  end
30
34
 
31
35
  def keys
32
- @data_store.keys.
33
- select { |k| k.start_with? @scope[''] }.
34
- map { |k| Key[k].remove_first @scope.sections.count }
36
+ data_store.keys
37
+ .select { |k| k.start_with? scope[''] }
38
+ .map { |k| Key[k].remove_first scope.sections.count }
35
39
  end
36
40
 
37
41
  def synchronize(key, &block)
@@ -41,20 +45,15 @@ module Asynchronic
41
45
  def connection_args
42
46
  [
43
47
  {
44
- data_store_class: @data_store.class,
45
- data_store_connection_args: @data_store.connection_args,
46
- scope: @scope
48
+ data_store_class: data_store.class,
49
+ data_store_connection_args: data_store.connection_args,
50
+ scope: scope
47
51
  }
48
52
  ]
49
53
  end
50
54
 
51
- def self.connect(*args)
52
- data_store = args[0][:data_store_class].connect *args[0][:data_store_connection_args]
53
- new data_store, args[0][:scope]
54
- end
55
-
56
55
  def to_s
57
- "#<#{self.class} @data_store=#{@data_store} @scope=#{@scope}>"
56
+ "#<#{self.class} @data_store=#{data_store} @scope=#{scope}>"
58
57
  end
59
58
 
60
59
  end
@@ -2,7 +2,7 @@ module Asynchronic
2
2
  class Environment
3
3
 
4
4
  attr_reader :queue_engine, :data_store, :notifier
5
-
5
+
6
6
  def initialize(queue_engine, data_store, notifier)
7
7
  @queue_engine = queue_engine
8
8
  @data_store = data_store
@@ -14,7 +14,7 @@ module Asynchronic
14
14
  end
15
15
 
16
16
  def default_queue
17
- queue(queue_engine.default_queue)
17
+ queue queue_engine.default_queue
18
18
  end
19
19
 
20
20
  def enqueue(msg, queue=nil)
@@ -28,7 +28,7 @@ module Asynchronic
28
28
  def load_process(id)
29
29
  Process.new self, id
30
30
  end
31
-
31
+
32
32
  def processes
33
33
  Process.all self
34
34
  end
@@ -1,9 +1,8 @@
1
1
  module Asynchronic
2
2
  class Error
3
3
 
4
- attr_reader :message
5
- attr_reader :backtrace
6
-
4
+ attr_reader :message, :backtrace
5
+
7
6
  def initialize(source)
8
7
  @message = source.respond_to?(:message) ? source.message : source.to_s
9
8
  @backtrace = source.respond_to?(:backtrace) ? source.backtrace : []
@@ -1,18 +1,6 @@
1
1
  module Asynchronic
2
2
  class Job
3
3
 
4
- def initialize(process)
5
- @process = process
6
- end
7
-
8
- def params
9
- @process.params
10
- end
11
-
12
- def result(reference)
13
- @process[reference].result
14
- end
15
-
16
4
  def self.queue(name=nil)
17
5
  name ? @queue = name : @queue
18
6
  end
@@ -23,6 +11,18 @@ module Asynchronic
23
11
  process.id
24
12
  end
25
13
 
14
+ def initialize(process)
15
+ @process = process
16
+ end
17
+
18
+ def params
19
+ process.params
20
+ end
21
+
22
+ def result(reference)
23
+ process[reference].result
24
+ end
25
+
26
26
  private
27
27
 
28
28
  attr_reader :process
@@ -8,23 +8,27 @@ module Asynchronic
8
8
  end
9
9
 
10
10
  def publish(pid, event, data=nil)
11
- @broadcaster.publish DataStore::Key[pid][event], data
11
+ broadcaster.publish DataStore::Key[pid][event], data
12
12
  end
13
13
 
14
14
  def subscribe(pid, event, &block)
15
- @broadcaster.subscribe DataStore::Key[pid][event] do |data|
15
+ broadcaster.subscribe DataStore::Key[pid][event] do |data|
16
16
  block.call data
17
17
  end
18
18
  end
19
19
 
20
20
  def unsubscribe(subscription_id)
21
- @broadcaster.unsubscribe subscription_id
21
+ broadcaster.unsubscribe subscription_id
22
22
  end
23
23
 
24
24
  def unsubscribe_all
25
- @broadcaster.unsubscribe_all
25
+ broadcaster.unsubscribe_all
26
26
  end
27
27
 
28
+ private
29
+
30
+ attr_reader :broadcaster
31
+
28
32
  end
29
33
  end
30
34
  end
@@ -19,6 +19,30 @@ module Asynchronic
19
19
 
20
20
  attr_reader :id
21
21
 
22
+ def self.create(environment, type, params={})
23
+ id = params.delete(:id) || SecureRandom.uuid
24
+
25
+ Asynchronic.logger.debug('Asynchronic') { "Created process #{type} - #{id} - #{params}" }
26
+
27
+ new(environment, id) do
28
+ self.type = type
29
+ self.name = (params.delete(:alias) || type).to_s
30
+ self.queue = params.delete(:queue) || type.queue || parent_queue
31
+ self.dependencies = Array(params.delete(:dependencies)) | Array(params.delete(:dependency)) | infer_dependencies(params)
32
+ self.params = params
33
+ self.data = {}
34
+ pending!
35
+ end
36
+ end
37
+
38
+ def self.all(environment)
39
+ environment.data_store.keys
40
+ .select { |k| k.sections.count == 2 && k.match(/created_at$/) }
41
+ .sort_by { |k| environment.data_store[k] }
42
+ .reverse
43
+ .map { |k| Process.new environment, k.remove_last }
44
+ end
45
+
22
46
  def initialize(environment, id, &block)
23
47
  @environment = environment
24
48
  @id = DataStore::Key[id]
@@ -84,9 +108,11 @@ module Asynchronic
84
108
  end
85
109
 
86
110
  def processes
87
- data_store.scoped(:processes).keys.
88
- select { |k| k.sections.count == 2 && k.match(/\|name$/) }.
89
- sort.map { |k| Process.new environment, id[:processes][k.remove_last] }
111
+ data_store.scoped(:processes)
112
+ .keys
113
+ .select { |k| k.sections.count == 2 && k.match(/\|name$/) }
114
+ .sort
115
+ .map { |k| Process.new environment, id[:processes][k.remove_last] }
90
116
  end
91
117
 
92
118
  def parent
@@ -107,11 +133,11 @@ module Asynchronic
107
133
 
108
134
  def dependencies
109
135
  return [] if parent.nil? || data_store[:dependencies].empty?
110
-
136
+
111
137
  parent_processes = parent.processes.each_with_object({}) do |process, hash|
112
138
  hash[process.name] = process
113
139
  end
114
-
140
+
115
141
  data_store[:dependencies].map { |d| parent_processes[d.to_s] }
116
142
  end
117
143
 
@@ -135,7 +161,7 @@ module Asynchronic
135
161
  wakeup_children
136
162
  end
137
163
  Asynchronic.logger.info('Asynchronic') { "Wakeup finalized #{type} (#{id})" }
138
-
164
+
139
165
  parent.wakeup if parent && finalized?
140
166
  end
141
167
 
@@ -151,29 +177,6 @@ module Asynchronic
151
177
  self.data = self.data.merge key => value
152
178
  end
153
179
 
154
- def self.create(environment, type, params={})
155
- id = params.delete(:id) || SecureRandom.uuid
156
-
157
- Asynchronic.logger.debug('Asynchronic') { "Created process #{type} - #{id} - #{params}" }
158
-
159
- new(environment, id) do
160
- self.type = type
161
- self.name = (params.delete(:alias) || type).to_s
162
- self.queue = params.delete(:queue) || type.queue || parent_queue
163
- self.dependencies = Array(params.delete(:dependencies)) | Array(params.delete(:dependency)) | infer_dependencies(params)
164
- self.params = params
165
- self.data = {}
166
- pending!
167
- end
168
- end
169
-
170
- def self.all(environment)
171
- environment.data_store.keys.
172
- select { |k| k.sections.count == 2 && k.match(/created_at$/) }.
173
- sort_by { |k| environment.data_store[k] }.reverse.
174
- map { |k| Process.new environment, k.remove_last }
175
- end
176
-
177
180
  private
178
181
 
179
182
  attr_reader :environment
@@ -194,10 +197,10 @@ module Asynchronic
194
197
 
195
198
  def status=(status)
196
199
  Asynchronic.logger.info('Asynchronic') { "#{status.to_s.capitalize} #{type} (#{id})" }
197
-
200
+
198
201
  data_store[:status] = status
199
202
  data_store[TIME_TRACKING_MAP[status]] = Time.now if TIME_TRACKING_MAP.key? status
200
-
203
+
201
204
  environment.notifier.publish id, :status_changed, status
202
205
  environment.notifier.publish id, :finalized if finalized?
203
206
  end
@@ -229,7 +232,7 @@ module Asynchronic
229
232
  self.result = job.call
230
233
  waiting!
231
234
  end
232
-
235
+
233
236
  rescue Exception => ex
234
237
  message = "Failed process #{type} (#{id})\n#{ex.class} #{ex.message}\n#{ex.backtrace.join("\n")}"
235
238
  Asynchronic.logger.error('Asynchronic') { message }