rukawa 0.4.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e90c29a3fb28cf675e9c34e2be8a10e642819f19
4
- data.tar.gz: f967d48f8faced73d118c7c752e3f1c28cb062fd
3
+ metadata.gz: b165a0824c474f449b62bd61beed9f96f3e11c39
4
+ data.tar.gz: bfaf7d5212c500d6084b07e4ca36baf745ab3efb
5
5
  SHA512:
6
- metadata.gz: c98fbb5ebf627f6133337c4d88e88556ce199eec049a65ca610f1faa0c74f932461086cbd0687d18a28fd52378c6cb9d2d4a15522423e113d7c1a90318c5f0a5
7
- data.tar.gz: 1c763aa161305e1d6db416be8f8237d40c4f432e9e3b8edefaad3002e179eece6da0bbc118913f1261ba4425e25cbb5ec5d77484ce16a588a6e71291f6489f27
6
+ metadata.gz: 0109918d72f5527124754e5ffdffa7c4ae97661a9c4a871e2c2620bc96e1514ee8affd3015a9e25d74134ee547c248eb41488f172333b8aa017ec1654e66cb4c
7
+ data.tar.gz: 2c662f7a992ae4172e4cb93d2ba5937bb25ef6aee7ddc9e5ef3cd929cc274fb5fec974035225c80a606cfd74d50a1d1138501594420f5e11c862902d6fb3bc1f
data/README.md CHANGED
@@ -262,6 +262,57 @@ var2: value2
262
262
  }
263
263
  ```
264
264
 
265
+ ### Semaphore
266
+
267
+ You can control concurrency consumption.
268
+
269
+ ```ruby
270
+ class Job < Rukawa::Job
271
+ set_resource_count 2
272
+
273
+ def run
274
+ # process
275
+ end
276
+ end
277
+ ```
278
+
279
+ This job use 2 concurrency. (this does not means that job use 2 threads)
280
+ If concurrency is less than jobs's resource count, resource count is set concurrency size.
281
+
282
+ ### Callback
283
+
284
+ - before\_run
285
+ - after\_run
286
+ - around\_run
287
+ - after\_fail
288
+
289
+ ```ruby
290
+ class Job < Rukawa::Job
291
+ before_run :wait_other_resource
292
+ after_run :notify_finish
293
+ around_run do |job, blk|
294
+ begin
295
+ setup_resource
296
+ blk.call
297
+ ensure
298
+ release_resource
299
+ end
300
+ end
301
+
302
+ def run
303
+ # process
304
+ end
305
+
306
+ def wait_other_resource
307
+ sleep(3)
308
+ end
309
+
310
+ def notify_finish
311
+ send_notification_to_slack
312
+ end
313
+ end
314
+ ```
315
+
265
316
  ### Config Example
266
317
 
267
318
  ```
data/lib/rukawa.rb CHANGED
@@ -2,12 +2,18 @@ require "concurrent"
2
2
 
3
3
  module Rukawa
4
4
  class << self
5
- def logger
6
- config.logger
5
+ def init
6
+ unless @initialized
7
+ @store = Concurrent::Hash.new
8
+ @executor = Concurrent::FixedThreadPool.new(config.concurrency)
9
+ @semaphore = Concurrent::Semaphore.new(config.concurrency)
10
+ @initialized = true
11
+ end
7
12
  end
13
+ attr_reader :store, :executor, :semaphore
8
14
 
9
- def store
10
- @store ||= Concurrent::Hash.new
15
+ def logger
16
+ config.logger
11
17
  end
12
18
 
13
19
  def configure
@@ -17,10 +23,6 @@ module Rukawa
17
23
  def config
18
24
  Configuration.instance
19
25
  end
20
-
21
- def executor
22
- @executor ||= Concurrent::FixedThreadPool.new(config.concurrency)
23
- end
24
26
  end
25
27
  end
26
28
 
data/lib/rukawa/job.rb CHANGED
@@ -3,15 +3,31 @@ require 'rukawa/abstract_job'
3
3
  require 'rukawa/dependency'
4
4
  require 'rukawa/state'
5
5
  require 'active_support/core_ext/class'
6
+ require 'active_support/callbacks'
6
7
 
7
8
  module Rukawa
8
9
  class Job < AbstractJob
10
+ include ActiveSupport::Callbacks
11
+ define_callbacks :run,
12
+ terminator: ->(_,result) { result == false },
13
+ skip_after_callbacks_if_terminated: true,
14
+ scope: [:kind, :name],
15
+ only: [:before, :around, :after]
16
+
17
+ define_callbacks :fail,
18
+ terminator: ->(_,result) { result == false },
19
+ skip_after_callbacks_if_terminated: true,
20
+ scope: [:kind, :name],
21
+ only: [:after]
22
+
9
23
  attr_accessor :in_comings, :out_goings
10
24
  attr_reader :state, :started_at, :finished_at, :variables
11
25
 
12
26
  class_attribute :retryable, :retry_limit, :retry_exception_type, :retry_wait, instance_writer: false
13
27
  class_attribute :dependency_type, instance_writer: false
28
+ class_attribute :resource_count, instance_reader: false, instance_writer: false
14
29
  self.dependency_type = Dependency::AllSuccess
30
+ self.resource_count = 1
15
31
 
16
32
  class << self
17
33
  def set_retryable(limit: 8, type: nil, wait: nil)
@@ -24,6 +40,50 @@ module Rukawa
24
40
  def set_dependency_type(name)
25
41
  self.dependency_type = Rukawa::Dependency.get(name)
26
42
  end
43
+
44
+ def set_resource_count(count)
45
+ self.resource_count = count
46
+ end
47
+
48
+ def before_run(*args, **options, &block)
49
+ set_callback :run, :before, *args, **options, &block
50
+ end
51
+
52
+ def after_run(*args, **options, &block)
53
+ options[:prepend] = true
54
+ conditional = ActiveSupport::Callbacks::Conditionals::Value.new { |v|
55
+ v != false
56
+ }
57
+ options[:if] = Array(options[:if]) << conditional
58
+ set_callback :run, :after, *args, **options, &block
59
+ end
60
+
61
+ def after_fail(*args, **options, &block)
62
+ options[:prepend] = true
63
+ conditional = ActiveSupport::Callbacks::Conditionals::Value.new { |v|
64
+ v != false
65
+ }
66
+ options[:if] = Array(options[:if]) << conditional
67
+ set_callback :fail, :after, *args, **options, &block
68
+ end
69
+
70
+ def around_run(*args, **options, &block)
71
+ set_callback :run, :around, *args, **options, &block
72
+ end
73
+ end
74
+
75
+ around_run :acquire_resource
76
+
77
+ around_run do |_, blk|
78
+ Rukawa.logger.info("Start #{self.class}")
79
+ blk.call
80
+ Rukawa.logger.info("Finish #{self.class}")
81
+ end
82
+
83
+ around_run do |_, blk|
84
+ set_state(:running)
85
+ blk.call
86
+ set_state(:finished)
27
87
  end
28
88
 
29
89
  def initialize(parent_job_net, variables)
@@ -69,13 +129,12 @@ module Rukawa
69
129
  set_state(:skipped)
70
130
  else
71
131
  check_dependencies(results)
72
- Rukawa.logger.info("Start #{self.class}")
73
- set_state(:running)
74
- run
75
- Rukawa.logger.info("Finish #{self.class}")
76
- set_state(:finished)
132
+ run_callbacks :run do
133
+ run
134
+ end
77
135
  end
78
136
  rescue => e
137
+ run_callbacks :fail
79
138
  handle_error(e)
80
139
  Rukawa.logger.error("Retry #{self.class}")
81
140
  retry
@@ -153,5 +212,16 @@ module Rukawa
153
212
  Rukawa.store[self.class] ||= Concurrent::Hash.new
154
213
  Rukawa.store[self.class][key] = value
155
214
  end
215
+
216
+ def resource_count
217
+ [self.class.resource_count, Rukawa.config.concurrency].min
218
+ end
219
+
220
+ def acquire_resource
221
+ Rukawa.semaphore.acquire(resource_count)
222
+ yield
223
+ ensure
224
+ Rukawa.semaphore.release(resource_count)
225
+ end
156
226
  end
157
227
  end
data/lib/rukawa/runner.rb CHANGED
@@ -16,6 +16,7 @@ module Rukawa
16
16
  end
17
17
 
18
18
  def run(batch_mode = false, refresh_interval = DEFAULT_REFRESH_INTERVAL)
19
+ Rukawa.init
19
20
  Rukawa.logger.info("=== Start Rukawa ===")
20
21
  futures = @root_job_net.dataflows.each(&:execute)
21
22
  until futures.all?(&:complete?)
@@ -1,3 +1,3 @@
1
1
  module Rukawa
2
- VERSION = "0.4.0"
2
+ VERSION = "0.5.0"
3
3
  end
@@ -4,6 +4,12 @@ module ExecuteLog
4
4
  end
5
5
  end
6
6
 
7
+ class Notifier
8
+ def after_run(job)
9
+ puts "#{job.name}:#{job.state.name}:#{job.elapsed_time_from}"
10
+ end
11
+ end
12
+
7
13
  class SampleJob < Rukawa::Job
8
14
  def run
9
15
  sleep rand(5)
@@ -13,6 +19,7 @@ end
13
19
 
14
20
  class Job1 < SampleJob
15
21
  set_description "Job1 description body"
22
+ after_run Notifier.new
16
23
 
17
24
  def run
18
25
  p variables unless variables.empty?
@@ -20,6 +27,10 @@ class Job1 < SampleJob
20
27
  end
21
28
  end
22
29
  class Job2 < SampleJob
30
+ after_fail do |job|
31
+ p "Job2 Failed!!! #{job.state.name}"
32
+ end
33
+
23
34
  def run
24
35
  raise "job2 error"
25
36
  end
@@ -29,6 +40,7 @@ end
29
40
  class Job4 < SampleJob
30
41
  # inherited by subclass
31
42
  set_dependency_type :one_success
43
+ set_resource_count 3
32
44
  end
33
45
  class Job5 < SampleJob
34
46
  # inherited by subclass
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rukawa
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - joker1007
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-04-30 00:00:00.000000000 Z
11
+ date: 2016-06-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -208,7 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
208
208
  version: '0'
209
209
  requirements: []
210
210
  rubyforge_project:
211
- rubygems_version: 2.5.1
211
+ rubygems_version: 2.6.4
212
212
  signing_key:
213
213
  specification_version: 4
214
214
  summary: Hyper simple job workflow engine