rukawa 0.7.0 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 8087ac19883cd6f2ab570ab99494db140cd86670
4
- data.tar.gz: 4b86cea0a91ba32c838a7b80fc7b6f7dc2334666
2
+ SHA256:
3
+ metadata.gz: f433f168b831f2b1c30a55129c711382b2519c5e2de79f669486cff353aa344a
4
+ data.tar.gz: 28c8226bf65d6f747723650a60786fd966e1e758a5d3145efcde7318e19208ea
5
5
  SHA512:
6
- metadata.gz: 074ca450e282d2a63198f357df4f0fb1b5c694c04734dea25b23ceb4ca4b8f5e7e5bb90807950ebc2b95c562a08503fa5af8627b61bd37fa266d15aa0223dbb3
7
- data.tar.gz: 4847ee20b6163067927180c183ee4f98686f35a8e86375d7035612f04d1dbfdb11ca549dac16c2f6ef2775c1211f48a53f670ded8ba067fd6e59c04d1fa47367
6
+ metadata.gz: 452a7bfd1726c3992f1788ae457a1208247ccdac4e6e7ce9b79b069c942c8d7a176ea1908ef481479e730d751089b9880ec39eb66b0449f018ab1fac9b6b9bda
7
+ data.tar.gz: 8a86b7ca28ea37638c0b92bb5747bd55f6ca347683d9285a368898c8a92a569b72b8e2a387a701e4292a1a887302880f4bade96c2808f061f726b0a2ddb8748f
@@ -0,0 +1,41 @@
1
+ name: test
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - '*'
7
+
8
+ pull_request:
9
+ # schedule:
10
+ # cron: '0 0 * * 0' # every Sunday at 00:00 (UTC)
11
+
12
+ jobs:
13
+ build:
14
+ runs-on: ubuntu-latest
15
+ name: Ruby ${{ matrix.ruby }}
16
+ strategy:
17
+ matrix:
18
+ ruby:
19
+ - '3.0'
20
+ - '3.1'
21
+ - '3.2'
22
+ services:
23
+ redis:
24
+ image: redis
25
+ options: >-
26
+ --health-cmd "redis-cli ping"
27
+ --health-interval 10s
28
+ --health-timeout 5s
29
+ --health-retries 5
30
+ ports:
31
+ - 6379:6379
32
+
33
+ steps:
34
+ - uses: actions/checkout@v2
35
+ - name: Set up Ruby
36
+ uses: ruby/setup-ruby@v1
37
+ with:
38
+ ruby-version: ${{ matrix.ruby }}
39
+ bundler-cache: true
40
+ - run: bundle exec rake spec
41
+
data/LICENSE.txt ADDED
@@ -0,0 +1,7 @@
1
+ Copyright 2016 Tomohiro Hashidate (joker1007)
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4
+
5
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6
+
7
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md CHANGED
@@ -1,7 +1,6 @@
1
- [![Stories in Ready](https://badge.waffle.io/joker1007/rukawa.png?label=ready&title=Ready)](https://waffle.io/joker1007/rukawa)
2
1
  # Rukawa
3
2
  [![Gem Version](https://badge.fury.io/rb/rukawa.svg)](https://badge.fury.io/rb/rukawa)
4
- [![Build Status](https://travis-ci.org/joker1007/rukawa.svg?branch=master)](https://travis-ci.org/joker1007/rukawa)
3
+ [![test](https://github.com/joker1007/rukawa/actions/workflows/test.yml/badge.svg)](https://github.com/joker1007/rukawa/actions/workflows/test.yml)
5
4
  [![Code Climate](https://codeclimate.com/github/joker1007/rukawa/badges/gpa.svg)](https://codeclimate.com/github/joker1007/rukawa)
6
5
 
7
6
  Rukawa = (流川)
@@ -44,7 +43,7 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
44
43
  ```
45
44
  % cd rukawa/sample
46
45
 
47
- # load ./jobs/**/*.rb, ./job_net/**/*.rb automatically
46
+ # load ./jobs/**/*.rb, ./job_nets/**/*.rb automatically
48
47
  % bundle exec rukawa run SampleJobNet -r 10 -c 10
49
48
  +----------------+----------+
50
49
  | Job | Status |
@@ -331,6 +330,37 @@ end
331
330
 
332
331
  see. [Rukawa::Configuration](https://github.com/joker1007/rukawa/blob/master/lib/rukawa/configuration.rb)
333
332
 
333
+ ### ActiveJob Integration
334
+
335
+ ```ruby
336
+ class SampleJobNet < Rukawa::JobNet
337
+ class << self
338
+ def dependencies
339
+ # Generate Wrapper class
340
+ wrapped1 = Rukawa::Wrapper::ActiveJob[ActiveJobSample1] # named Rukawa::Wrapper::ActiveJobSample1Wrapper
341
+ wrapped2 = Rukawa::Wrapper::ActiveJob[ActiveJobSample2] # named Rukawa::Wrapper::ActiveJobSample2Wrapper
342
+ {
343
+ Job1 => [],
344
+ wrapped1 => [Job1],
345
+ wrapped2 => [wrapped1],
346
+ }
347
+ end
348
+ end
349
+ end
350
+ ```
351
+
352
+ And write config to define status store for tracking remote job status"
353
+
354
+ ```ruby
355
+ redis_host = ENV["REDIS_HOST"] || "localhost:6379"
356
+ Rukawa.configure do |c|
357
+ c.status_store = ActiveSupport::Cache::RedisStore.new(redis_host)
358
+ c.status_expire_duration = 72 * 60 * 60 # default is 24 hours
359
+ end
360
+ ```
361
+
362
+ __Caution: When rukawa runs wrapper job, base ActiveJob class includes hook modules automatically in order to track job running status.__
363
+
334
364
  ### help
335
365
  ```
336
366
  % bundle exec rukawa help run
@@ -392,7 +422,7 @@ Output jobnet graph. If JOB_NET is set, simulate resumed job sequence
392
422
 
393
423
  ```ruby
394
424
  currency = 4
395
- job_net = YourJobNetClass.new(nil, {"var1" => "value1"}, Context.new(currency))
425
+ job_net = YourJobNetClass.new(variables: {"var1" => "value1"})
396
426
  promise = job_net.run do
397
427
  puts "Job Running"
398
428
  end
@@ -417,4 +447,3 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
417
447
  ## Contributing
418
448
 
419
449
  Bug reports and pull requests are welcome on GitHub at https://github.com/joker1007/rukawa.
420
-
@@ -26,7 +26,7 @@ module Rukawa
26
26
  end
27
27
 
28
28
  def name
29
- self.class.to_s
29
+ self.class.name || self.class.to_s
30
30
  end
31
31
 
32
32
  def inspect
@@ -0,0 +1,22 @@
1
+ require 'rukawa/job'
2
+
3
+ module Rukawa
4
+ module Builtins
5
+ class Base < ::Rukawa::Job
6
+ class << self
7
+ def [](**params)
8
+ Class.new(self) do
9
+ handle_parameters(params)
10
+
11
+ def self.name
12
+ super || "#{superclass.name}_#{object_id}"
13
+ end
14
+ end
15
+ end
16
+
17
+ def handle_parameters(**params)
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,60 @@
1
+ require 'rukawa/builtins/shell'
2
+
3
+ module Rukawa
4
+ module Builtins
5
+ class Embulk < Shell
6
+ class_attribute :config, :embulk_bin, :embulk_bundle, :embulk_vm_options, :jvm_options, :preview
7
+
8
+ self.embulk_bin = "embulk"
9
+ self.embulk_bundle = nil
10
+ self.embulk_vm_options = []
11
+ self.jvm_options = []
12
+ self.preview = false
13
+
14
+ class << self
15
+ def handle_parameters(config:, embulk_bin: nil, embulk_bundle: nil, embulk_vm_options: nil, jvm_options: nil, stdout: nil, stderr: nil, env: nil, chdir: nil, preview: false)
16
+ self.config = config
17
+ self.embulk_bin = embulk_bin if embulk_bin
18
+ self.embulk_bundle = embulk_bundle if embulk_bundle
19
+ self.embulk_vm_options = embulk_vm_options if embulk_vm_options
20
+ self.jvm_options = jvm_options if jvm_options
21
+ self.stdout = stdout if stdout
22
+ self.stderr = stderr if stderr
23
+ self.env = env if env
24
+ self.chdir = chdir if chdir
25
+ self.preview = preview
26
+ end
27
+ end
28
+
29
+ def run
30
+ process = -> do
31
+ if preview
32
+ stdout.puts "Config:\n#{File.read(config)}"
33
+ cmds = [embulk_bin, "preview", *embulk_bundle, config].compact
34
+ stdout.puts cmds.join(" ")
35
+ else
36
+ cmds = [embulk_bin, *embulk_vm_options, *jvm_options, "run", *embulk_bundle, config].compact
37
+ stdout.puts cmds.join(" ")
38
+ end
39
+
40
+ stdout.flush
41
+
42
+ cmdenv = env || {}
43
+ opts = chdir ? {chdir: chdir} : {}
44
+ result, log = execute_command(cmds, cmdenv, opts)
45
+
46
+ unless result.success?
47
+ next if log =~ /NoSampleException/
48
+ raise "embulk error"
49
+ end
50
+ end
51
+
52
+ if defined?(Bundler)
53
+ Bundler.with_clean_env(&process)
54
+ else
55
+ process.call
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,72 @@
1
+ require 'rukawa/builtins/base'
2
+ require 'active_support/core_ext/class'
3
+ require 'open3'
4
+
5
+ module Rukawa
6
+ module Builtins
7
+ class Shell < Base
8
+ class_attribute :command, :args, :env, :chdir, :stdout, :stderr
9
+
10
+ self.args = []
11
+ self.stdout = $stdout
12
+ self.stderr = $stderr
13
+
14
+ class << self
15
+ def handle_parameters(command:, args: [], env: nil, chdir: nil, stdout: nil, stderr: nil, **rest)
16
+ self.command = command
17
+ self.args = args
18
+ self.env = env if env
19
+ self.chdir = chdir if chdir
20
+ self.stdout = stdout if stdout
21
+ self.stderr = stderr if stderr
22
+ end
23
+ end
24
+
25
+ def run
26
+ cmdenv = env || {}
27
+ opts = chdir ? {chdir: chdir} : {}
28
+
29
+ if defined?(Bundler)
30
+ result, log = nil
31
+ Bundler.with_clean_env do
32
+ result, log = execute_command([command, *args], cmdenv, opts)
33
+ end
34
+ else
35
+ result, log = execute_command([command, *args], cmdenv, opts)
36
+ end
37
+
38
+ unless result.success?
39
+ raise "command error"
40
+ end
41
+ end
42
+
43
+ def execute_command(command, env, opts)
44
+ log = "".dup
45
+ result = Open3.popen3(env, *command, opts) do |stdin, out, err, wait_th|
46
+ stdin.close
47
+ until out.eof? && err.eof?
48
+ rs, = IO.select([out, err])
49
+ rs.each do |rio|
50
+ line = rio.gets
51
+ if line
52
+ log << line
53
+ if rio == out
54
+ stdout.write(line)
55
+ else
56
+ stderr.write(line)
57
+ end
58
+ end
59
+ end
60
+ end
61
+
62
+ wait_th.value
63
+ end
64
+
65
+ stdout.flush
66
+ stderr.flush unless stdout == stderr
67
+
68
+ return result, log
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,208 @@
1
+ require 'rukawa/builtins/base'
2
+ require 'timeout'
3
+
4
+ module Rukawa
5
+ module Builtins
6
+ class Waiter < Base
7
+ class_attribute :timeout, :poll_interval
8
+
9
+ self.timeout = 1800
10
+ self.poll_interval = 1
11
+
12
+ class << self
13
+ def handle_parameters(timeout: nil, poll_interval: nil, **rest)
14
+ self.timeout = timeout if timeout
15
+ self.poll_interval = poll_interval if poll_interval
16
+ end
17
+ end
18
+
19
+ def run
20
+ Timeout.timeout(timeout) do
21
+ wait_until do
22
+ fetch_condition
23
+ end
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ def wait_until
30
+ until yield
31
+ sleep poll_interval
32
+ end
33
+ end
34
+
35
+ def fetch_condition
36
+ raise NotImplementedError
37
+ end
38
+ end
39
+
40
+ class SleepWaiter < Waiter
41
+ class_attribute :sec
42
+
43
+ class << self
44
+ def handle_parameters(sec:, **rest)
45
+ self.sec = sec
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def fetch_condition
52
+ sleep sec
53
+ end
54
+ end
55
+
56
+ class LocalFileWaiter < Waiter
57
+ class_attribute :path, :if_modified_since, :if_unmodified_since
58
+
59
+ class << self
60
+ def handle_parameters(path:, if_modified_since: nil, if_unmodified_since: nil, **rest)
61
+ self.path = path
62
+ self.if_modified_since = if_modified_since if if_modified_since
63
+ self.if_unmodified_since = if_unmodified_since if if_unmodified_since
64
+ super(**rest)
65
+ end
66
+ end
67
+
68
+ private
69
+
70
+ def fetch_condition
71
+ if path.respond_to?(:all?)
72
+ get_stat_and_check_condition(p)
73
+ path.all?(&method(:get_stat_and_check_condition))
74
+ else
75
+ get_stat_and_check_condition(path)
76
+ end
77
+ end
78
+
79
+ def get_stat_and_check_condition(path)
80
+ stat = File.stat(path)
81
+
82
+ if if_modified_since
83
+ stat.mtime > if_modified_since
84
+ elsif if_unmodified_since
85
+ stat.mtime <= if_unmodified_since
86
+ else
87
+ true
88
+ end
89
+ rescue
90
+ false
91
+ end
92
+ end
93
+
94
+ class S3Waiter < Waiter
95
+ class_attribute :url, :aws_access_key_id, :aws_secret_access_key, :region, :if_modified_since, :if_unmodified_since
96
+
97
+ class << self
98
+ def handle_parameters(url:, aws_access_key_id: nil, aws_secret_access_key: nil, region: nil, if_modified_since: nil, if_unmodified_since: nil, **rest)
99
+ require 'aws-sdk'
100
+
101
+ self.url = url
102
+ self.aws_access_key_id = aws_access_key_id if aws_access_key_id
103
+ self.aws_secret_access_key = aws_secret_access_key if aws_secret_access_key
104
+ self.region = region if region
105
+ self.if_modified_since = if_modified_since if if_modified_since
106
+ self.if_unmodified_since = if_unmodified_since if if_unmodified_since
107
+ super(**rest)
108
+ end
109
+ end
110
+
111
+ private
112
+
113
+ def fetch_condition
114
+ opts = {if_modified_since: if_modified_since, if_unmodified_since: if_unmodified_since}.reject do |_, v|
115
+ v.nil?
116
+ end
117
+
118
+ if url.respond_to?(:all?)
119
+ url.all? do |u|
120
+ s3url = URI.parse(u)
121
+ client.head_object(bucket: s3url.host, key: s3url.path[1..-1], **opts) rescue false
122
+ end
123
+ else
124
+ s3url = URI.parse(url)
125
+ client.head_object(bucket: s3url.host, key: s3url.path[1..-1], **opts) rescue false
126
+ end
127
+ end
128
+
129
+ def client
130
+ return @client if @client
131
+
132
+ if aws_secret_access_key || aws_secret_access_key || region
133
+ options = {access_key_id: aws_access_key_id, secret_access_key: aws_secret_access_key, region: region}.reject do |_, v|
134
+ v.nil?
135
+ end
136
+ @client = Aws::S3::Client.new(options)
137
+ else
138
+ @client = Aws::S3::Client.new
139
+ end
140
+ end
141
+ end
142
+
143
+ class GCSWaiter < Waiter
144
+ class_attribute :url, :json_key, :if_modified_since, :if_unmodified_since
145
+
146
+ class << self
147
+ def handle_parameters(url:, json_key: nil, if_modified_since: nil, if_unmodified_since: nil, **rest)
148
+ require 'google/apis/storage_v1'
149
+ require 'googleauth'
150
+
151
+ self.url = url
152
+ self.json_key = json_key if json_key
153
+ self.if_modified_since = if_modified_since if if_modified_since
154
+ self.if_unmodified_since = if_unmodified_since if if_unmodified_since
155
+ super(**rest)
156
+ end
157
+ end
158
+
159
+ private
160
+
161
+ def fetch_condition
162
+ if url.respond_to?(:all?)
163
+ url.all? do |u|
164
+ get_object_and_check_condition(URI.parse(u))
165
+ end
166
+ else
167
+ get_object_and_check_condition(URI.parse(url))
168
+ end
169
+ end
170
+
171
+ def get_object_and_check_condition(url)
172
+ obj = client.get_object(url.host, url.path[1..-1])
173
+ if if_modified_since
174
+ obj.updated.to_time > if_modified_since
175
+ elsif if_unmodified_since
176
+ obj.updated.to_time <= if_unmodified_since
177
+ else
178
+ true
179
+ end
180
+ rescue
181
+ false
182
+ end
183
+
184
+ def client
185
+ return @client if @client
186
+
187
+ client = Google::Apis::StorageV1::StorageService.new
188
+ scope = "https://www.googleapis.com/auth/devstorage.read_only"
189
+
190
+ if json_key
191
+ begin
192
+ JSON.parse(json_key)
193
+ key = StringIO.new(json_key)
194
+ client.authorization = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: scope)
195
+ rescue JSON::ParserError
196
+ key = json_key
197
+ File.open(json_key) do |f|
198
+ client.authorization = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: f, scope: scope)
199
+ end
200
+ end
201
+ else
202
+ client.authorization = Google::Auth.get_application_default([scope])
203
+ end
204
+ client
205
+ end
206
+ end
207
+ end
208
+ end
data/lib/rukawa/cli.rb CHANGED
@@ -35,7 +35,7 @@ module Rukawa
35
35
 
36
36
  job_net_class = get_class(job_net_name)
37
37
  job_classes = job_name.map { |name| get_class(name) }
38
- job_net = job_net_class.new(nil, variables, Context.new, *job_classes)
38
+ job_net = job_net_class.new(variables: variables, resume_job_classes: job_classes)
39
39
  result = Runner.run(job_net, options[:batch], options[:refresh_interval])
40
40
 
41
41
  if options[:dot]
@@ -60,7 +60,7 @@ module Rukawa
60
60
 
61
61
  job_net_class = get_class(job_net_name)
62
62
  job_classes = job_name.map { |name| get_class(name) }
63
- job_net = job_net_class.new(nil, {}, Context.new, *job_classes)
63
+ job_net = job_net_class.new(resume_job_classes: job_classes)
64
64
  job_net.output_dot(options[:output], format: options[:format])
65
65
  end
66
66
 
@@ -75,7 +75,7 @@ module Rukawa
75
75
 
76
76
  job_classes = job_name.map { |name| get_class(name) }
77
77
  job_net_class = anonymous_job_net_class(*job_classes)
78
- job_net = job_net_class.new(nil, variables, Context.new)
78
+ job_net = job_net_class.new
79
79
  result = Runner.run(job_net, options[:batch], options[:refresh_interval])
80
80
 
81
81
  if options[:dot]
@@ -131,8 +131,6 @@ module Rukawa
131
131
  c.logger = Syslog::Logger.new('rukawa')
132
132
  elsif options[:log]
133
133
  c.logger = Logger.new(options[:log])
134
- else
135
- c.logger ||= Logger.new('./rukawa.log');
136
134
  end
137
135
  end
138
136
  end
@@ -6,13 +6,15 @@ require 'concurrent'
6
6
  module Rukawa
7
7
  class Configuration < Delegator
8
8
  include Singleton
9
- attr_accessor :logger
10
9
 
11
10
  def initialize
12
11
  @config = OpenStruct.new(
13
12
  concurrency: Concurrent.processor_count,
14
13
  dot_command: "dot",
15
- job_dirs: [File.join(Dir.pwd, "job_nets"), File.join(Dir.pwd, "jobs")]
14
+ job_dirs: [File.join(Dir.pwd, "job_nets"), File.join(Dir.pwd, "jobs")],
15
+ status_store: nil,
16
+ status_expire_duration: 24 * 60 * 60,
17
+ logger: Logger.new('./rukawa.log')
16
18
  )
17
19
  @config.graph = GraphConfig.new.tap { |c| c.rankdir = "LR" }
18
20
  end
data/lib/rukawa/dag.rb CHANGED
@@ -18,7 +18,7 @@ module Rukawa
18
18
  deps = tsortable_hash(dependencies).tsort
19
19
 
20
20
  deps.each do |job_class|
21
- job = job_class.new(job_net, variables, context)
21
+ job = job_class.new(variables: variables, context: context, parent_job_net: job_net)
22
22
  @nodes << job
23
23
  @jobs << job if job.is_a?(Job)
24
24
 
@@ -92,6 +92,7 @@ module Rukawa
92
92
  private
93
93
 
94
94
  def tsortable_hash(hash)
95
+ ensure_dependencies_have_all_jobs_as_key!(hash)
95
96
  class << hash
96
97
  include TSort
97
98
  alias :tsort_each_node :each_key
@@ -102,6 +103,14 @@ module Rukawa
102
103
  hash
103
104
  end
104
105
 
106
+ def ensure_dependencies_have_all_jobs_as_key!(hash)
107
+ hash.values.each do |parents|
108
+ parents.each do |j|
109
+ hash[j] ||= []
110
+ end
111
+ end
112
+ end
113
+
105
114
  class Edge
106
115
  attr_reader :from, :to, :cluster
107
116
 
data/lib/rukawa/job.rb CHANGED
@@ -70,6 +70,17 @@ module Rukawa
70
70
  def around_run(*args, **options, &block)
71
71
  set_callback :run, :around, *args, **options, &block
72
72
  end
73
+
74
+ def wrappers
75
+ @@wrappers ||= {}
76
+ end
77
+
78
+ def wrapper_for(*classes)
79
+ classes.each do |c|
80
+ raise "Wrapper for #{c} is already defined" if wrappers[c]
81
+ wrappers[c] = self
82
+ end
83
+ end
73
84
  end
74
85
 
75
86
  around_run do |_, blk|
@@ -84,7 +95,7 @@ module Rukawa
84
95
  set_state(:finished)
85
96
  end
86
97
 
87
- def initialize(parent_job_net, variables, context)
98
+ def initialize(variables: {}, context: Context.new, parent_job_net: nil)
88
99
  @parent_job_net = parent_job_net
89
100
  @variables = variables
90
101
  @context = context
@@ -150,9 +161,9 @@ module Rukawa
150
161
 
151
162
  def to_dot_def
152
163
  if state == Rukawa::State::Waiting
153
- "#{name};\n"
164
+ "\"#{name}\";\n"
154
165
  else
155
- "#{name} [style = filled,fillcolor = #{state.color}];\n"
166
+ "\"#{name}\" [style = filled,fillcolor = #{state.color}];\n"
156
167
  end
157
168
  end
158
169
 
@@ -218,6 +229,15 @@ module Rukawa
218
229
  @context.store[self.class][key] = value
219
230
  end
220
231
 
232
+ def fetch(job, key)
233
+ job = job.is_a?(String) ? Object.const_get(job) : job
234
+ raise TypeError, "job must be a Class" unless job.is_a?(Class)
235
+
236
+ if @context.store[job]
237
+ @context.store[job][key]
238
+ end
239
+ end
240
+
221
241
  def acquire_resource
222
242
  @context.semaphore.acquire(resource_count) if resource_count > 0
223
243
  yield
@@ -11,7 +11,7 @@ module Rukawa
11
11
  end
12
12
  end
13
13
 
14
- def initialize(parent_job_net, variables, context, *resume_job_classes)
14
+ def initialize(variables: {}, context: Context.new, parent_job_net: nil, resume_job_classes: [])
15
15
  @parent_job_net = parent_job_net
16
16
  @variables = variables
17
17
  @context = context
@@ -0,0 +1,46 @@
1
+ module Rukawa
2
+ module Remote
3
+ class StatusStore
4
+ ENQUEUED = "enqueued".freeze
5
+ PERFORMING = "performing".freeze
6
+ COMPLETED = "completed".freeze
7
+ FAILED = "failed".freeze
8
+
9
+ # default expire duration is 24 hours.
10
+ def initialize(job_id:, expire_duration: Rukawa.config.status_expire_duration)
11
+ @job_id = job_id
12
+ @expire_duration = expire_duration
13
+ end
14
+
15
+ def fetch
16
+ Rukawa.config.status_store.fetch(store_key)
17
+ end
18
+
19
+ def enqueued
20
+ Rukawa.config.status_store.write(store_key, ENQUEUED, expires_in: @expire_duration)
21
+ end
22
+
23
+ def performing
24
+ Rukawa.config.status_store.write(store_key, PERFORMING, expires_in: @expire_duration)
25
+ end
26
+
27
+ def completed
28
+ Rukawa.config.status_store.write(store_key, COMPLETED, expires_in: @expire_duration)
29
+ end
30
+
31
+ def failed
32
+ Rukawa.config.status_store.write(store_key, FAILED, expires_in: @expire_duration)
33
+ end
34
+
35
+ def delete
36
+ Rukawa.config.status_store.delete(store_key)
37
+ end
38
+
39
+ private
40
+
41
+ def store_key
42
+ "rukawa.remote_job.status.#{@job_id}"
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,9 @@
1
+ module Rukawa
2
+ module Remote
3
+ class << self
4
+ def store
5
+ Rukawa.config.status_store
6
+ end
7
+ end
8
+ end
9
+ end
@@ -1,3 +1,3 @@
1
1
  module Rukawa
2
- VERSION = "0.7.0"
2
+ VERSION = "0.9.0"
3
3
  end
@@ -0,0 +1,75 @@
1
+ require 'rukawa/job'
2
+ require 'rukawa/wrapper'
3
+ require 'rukawa/remote/status_store'
4
+
5
+ module Rukawa
6
+ module Wrapper
7
+ module ActiveJob
8
+ def self.[](job_class)
9
+ raise "Please set Rukawa.config.status_store subclass of ActiveSupport::Cache::Store" unless Rukawa.config.status_store
10
+ @wrapper_classes ||= {}
11
+ return @wrapper_classes[job_class] if @wrapper_classes[job_class]
12
+
13
+ wrapper = Class.new(Rukawa::Job) do
14
+ set_resource_count 0
15
+
16
+ define_singleton_method(:origin_class) do
17
+ job_class
18
+ end
19
+
20
+ def initialize(variables: {}, context: Context.new, parent_job_net: nil)
21
+ super
22
+ @job_class = self.class.origin_class
23
+ end
24
+
25
+ def run
26
+ @job_class.include(Hooks) unless @job_class.include?(Hooks)
27
+ @job_class.prepend(HooksForFailure) unless @job_class.include?(HooksForFailure)
28
+ job = @job_class.perform_later
29
+
30
+ status_store = Rukawa::Remote::StatusStore.new(job_id: job.job_id)
31
+ finish_statuses = [Rukawa::Remote::StatusStore::COMPLETED, Rukawa::Remote::StatusStore::FAILED]
32
+ until finish_statuses.include?(last_status = status_store.fetch)
33
+ sleep 0.1
34
+ end
35
+
36
+ status_store.delete
37
+
38
+ raise WrappedJobError if last_status == Rukawa::Remote::StatusStore::FAILED
39
+ end
40
+ end
41
+
42
+ @wrapper_classes[job_class] = wrapper
43
+ Rukawa::Wrapper.const_set("#{job_class.to_s.gsub(/::/, "_")}Wrapper", wrapper)
44
+ wrapper
45
+ end
46
+ end
47
+
48
+ module Hooks
49
+ def self.included(base)
50
+ base.class_eval do
51
+ before_enqueue { status_store.enqueued }
52
+
53
+ before_perform { status_store.performing }
54
+
55
+ after_perform { status_store.completed }
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ def status_store
62
+ @status_store ||= Rukawa::Remote::StatusStore.new(job_id: job_id)
63
+ end
64
+ end
65
+
66
+ module HooksForFailure
67
+ def perform(*args)
68
+ super
69
+ rescue Exception
70
+ status_store.failed
71
+ raise
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,5 @@
1
+ module Rukawa
2
+ module Wrapper
3
+ class WrappedJobError < StandardError; end
4
+ end
5
+ end
data/lib/rukawa.rb CHANGED
@@ -33,3 +33,4 @@ require 'rukawa/configuration'
33
33
  require 'rukawa/job_net'
34
34
  require 'rukawa/job'
35
35
  require 'rukawa/dag'
36
+ require 'rukawa/wrapper/active_job'
data/rukawa.gemspec CHANGED
@@ -18,15 +18,20 @@ Gem::Specification.new do |spec|
18
18
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
19
19
  spec.require_paths = ["lib"]
20
20
 
21
- spec.add_runtime_dependency "activesupport", ">= 4"
21
+ spec.add_runtime_dependency "activesupport", ">= 6"
22
+ spec.add_runtime_dependency "redis"
22
23
  spec.add_runtime_dependency "concurrent-ruby"
23
24
  spec.add_runtime_dependency "thor"
24
25
  spec.add_runtime_dependency "terminal-table"
25
26
  spec.add_runtime_dependency "paint"
26
27
 
27
- spec.add_development_dependency "bundler", "~> 1.11"
28
- spec.add_development_dependency "rake", "~> 10.0"
28
+ spec.add_development_dependency "bundler", "~> 2.0"
29
+ spec.add_development_dependency "rake", "~> 13.0"
29
30
  spec.add_development_dependency "rspec", "~> 3.0"
30
31
  spec.add_development_dependency "rspec-power_assert"
31
32
  spec.add_development_dependency "rspec-parameterized"
33
+ spec.add_development_dependency "activejob"
34
+ spec.add_development_dependency "sucker_punch"
35
+ spec.add_development_dependency "aws-sdk", "~> 2.0"
36
+ spec.add_development_dependency "google-api-client", "~> 0.9"
32
37
  end
@@ -2,8 +2,6 @@ class InnerJobNet < Rukawa::JobNet
2
2
  class << self
3
3
  def dependencies
4
4
  {
5
- InnerJob3 => [],
6
- InnerJob1 => [],
7
5
  InnerJob2 => [InnerJob1, InnerJob3],
8
6
  }
9
7
  end
@@ -14,7 +12,6 @@ class InnerJobNet2 < Rukawa::JobNet
14
12
  class << self
15
13
  def dependencies
16
14
  {
17
- InnerJob4 => [],
18
15
  InnerJob5 => [InnerJob4],
19
16
  InnerJob6 => [InnerJob4, InnerJob5],
20
17
  }
@@ -26,8 +23,6 @@ class InnerJobNet3 < Rukawa::JobNet
26
23
  class << self
27
24
  def dependencies
28
25
  {
29
- InnerJob7 => [],
30
- InnerJob8 => [],
31
26
  InnerJob9 => [InnerJob7, InnerJob8],
32
27
  InnerJob10 => [InnerJob7, InnerJob8],
33
28
  }
@@ -39,9 +34,6 @@ class InnerJobNet4 < Rukawa::JobNet
39
34
  class << self
40
35
  def dependencies
41
36
  {
42
- InnerJob11 => [],
43
- InnerJob12 => [],
44
- InnerJob13 => [],
45
37
  NestedJobNet => [InnerJob11, InnerJob12],
46
38
  }
47
39
  end
@@ -52,7 +44,6 @@ class NestedJobNet < Rukawa::JobNet
52
44
  class << self
53
45
  def dependencies
54
46
  {
55
- NestedJob1 => [],
56
47
  NestedJob2 => [NestedJob1],
57
48
  }
58
49
  end
@@ -62,9 +53,13 @@ end
62
53
  class SampleJobNet < Rukawa::JobNet
63
54
  class << self
64
55
  def dependencies
56
+ wrapped1 = Rukawa::Wrapper::ActiveJob[ActiveJobSample1]
57
+ wrapped2 = Rukawa::Wrapper::ActiveJob[ActiveJobSample2]
65
58
  {
66
59
  Job1 => [],
67
- Job2 => [Job1], Job3 => [Job1],
60
+ wrapped1 => [Job1],
61
+ wrapped2 => [wrapped1],
62
+ Job2 => [Job1], Job3 => [Job1, wrapped1],
68
63
  Job4 => [Job2, Job3],
69
64
  InnerJobNet => [Job3],
70
65
  Job8 => [InnerJobNet],
data/sample/jobnet.png CHANGED
Binary file
@@ -0,0 +1,19 @@
1
+ require 'active_job'
2
+
3
+ class ActiveJobSample1 < ActiveJob::Base
4
+ queue_as :default
5
+
6
+ def perform
7
+ sleep 10
8
+ p "active_job1"
9
+ end
10
+ end
11
+
12
+ class ActiveJobSample2 < ActiveJob::Base
13
+ queue_as :default
14
+
15
+ def perform
16
+ sleep 10
17
+ raise "active_job2 is failed"
18
+ end
19
+ end
data/sample/result.dot CHANGED
@@ -1,31 +1,33 @@
1
1
  digraph "SampleJobNet" {
2
2
  label = "SampleJobNet";
3
3
  graph [rankdir = LR,nodesep = 0.8,concentrate = true];
4
- Job1 [style = filled,fillcolor = green];
5
- Job2 [style = filled,fillcolor = red];
6
- Job3 [style = filled,fillcolor = green];
7
- Job4 [style = filled,fillcolor = green];
4
+ "Job1" [style = filled,fillcolor = green];
5
+ "Rukawa::Wrapper::ActiveJobSample1Wrapper" [style = filled,fillcolor = green];
6
+ "Rukawa::Wrapper::ActiveJobSample2Wrapper" [style = filled,fillcolor = red];
7
+ "Job2" [style = filled,fillcolor = red];
8
+ "Job3" [style = filled,fillcolor = green];
9
+ "Job4" [style = filled,fillcolor = green];
8
10
  subgraph "cluster_InnerJobNet" {
9
11
  label = "InnerJobNet";
10
12
  graph [rankdir = LR,nodesep = 0.8,concentrate = true];
11
13
  color = blue;
12
- InnerJob3 [style = filled,fillcolor = green];
13
- InnerJob1 [style = filled,fillcolor = green];
14
- InnerJob2 [style = filled,fillcolor = red];
14
+ "InnerJob3" [style = filled,fillcolor = green];
15
+ "InnerJob1" [style = filled,fillcolor = green];
16
+ "InnerJob2" [style = filled,fillcolor = red];
15
17
  "InnerJob1" -> "InnerJob2";
16
18
  "InnerJob3" -> "InnerJob2";
17
19
  }
18
- Job8 [style = filled,fillcolor = magenta];
19
- Job5 [style = filled,fillcolor = red];
20
- Job6 [style = filled,fillcolor = green];
21
- Job7 [style = filled,fillcolor = green];
20
+ "Job8" [style = filled,fillcolor = magenta];
21
+ "Job5" [style = filled,fillcolor = red];
22
+ "Job6" [style = filled,fillcolor = green];
23
+ "Job7" [style = filled,fillcolor = green];
22
24
  subgraph "cluster_InnerJobNet2" {
23
25
  label = "InnerJobNet2";
24
26
  graph [rankdir = LR,nodesep = 0.8,concentrate = true];
25
27
  color = blue;
26
- InnerJob4 [style = filled,fillcolor = green];
27
- InnerJob5 [style = filled,fillcolor = yellow];
28
- InnerJob6 [style = filled,fillcolor = magenta];
28
+ "InnerJob4" [style = filled,fillcolor = green];
29
+ "InnerJob5" [style = filled,fillcolor = yellow];
30
+ "InnerJob6" [style = filled,fillcolor = magenta];
29
31
  "InnerJob4" -> "InnerJob5";
30
32
  "InnerJob4" -> "InnerJob6";
31
33
  "InnerJob5" -> "InnerJob6";
@@ -34,10 +36,10 @@ subgraph "cluster_InnerJobNet3" {
34
36
  label = "InnerJobNet3";
35
37
  graph [rankdir = LR,nodesep = 0.8,concentrate = true];
36
38
  color = blue;
37
- InnerJob7 [style = filled,fillcolor = magenta];
38
- InnerJob8 [style = filled,fillcolor = magenta];
39
- InnerJob9 [style = filled,fillcolor = magenta];
40
- InnerJob10 [style = filled,fillcolor = magenta];
39
+ "InnerJob7" [style = filled,fillcolor = magenta];
40
+ "InnerJob8" [style = filled,fillcolor = magenta];
41
+ "InnerJob9" [style = filled,fillcolor = magenta];
42
+ "InnerJob10" [style = filled,fillcolor = magenta];
41
43
  "InnerJob7" -> "InnerJob9";
42
44
  "InnerJob8" -> "InnerJob9";
43
45
  "InnerJob7" -> "InnerJob10";
@@ -47,22 +49,25 @@ subgraph "cluster_InnerJobNet4" {
47
49
  label = "InnerJobNet4";
48
50
  graph [rankdir = LR,nodesep = 0.8,concentrate = true];
49
51
  color = blue;
50
- InnerJob11 [style = filled,fillcolor = magenta];
51
- InnerJob12 [style = filled,fillcolor = magenta];
52
- InnerJob13 [style = filled,fillcolor = magenta];
52
+ "InnerJob11" [style = filled,fillcolor = magenta];
53
+ "InnerJob12" [style = filled,fillcolor = magenta];
54
+ "InnerJob13" [style = filled,fillcolor = magenta];
53
55
  subgraph "cluster_NestedJobNet" {
54
56
  label = "NestedJobNet";
55
57
  graph [rankdir = LR,nodesep = 0.8,concentrate = true];
56
58
  color = blue;
57
- NestedJob1 [style = filled,fillcolor = magenta];
58
- NestedJob2 [style = filled,fillcolor = magenta];
59
+ "NestedJob1" [style = filled,fillcolor = magenta];
60
+ "NestedJob2" [style = filled,fillcolor = magenta];
59
61
  "NestedJob1" -> "NestedJob2";
60
62
  }
61
63
  "InnerJob11" -> "NestedJob1";
62
64
  "InnerJob12" -> "NestedJob1";
63
65
  }
66
+ "Job1" -> "Rukawa::Wrapper::ActiveJobSample1Wrapper";
67
+ "Rukawa::Wrapper::ActiveJobSample1Wrapper" -> "Rukawa::Wrapper::ActiveJobSample2Wrapper";
64
68
  "Job1" -> "Job2";
65
69
  "Job1" -> "Job3";
70
+ "Rukawa::Wrapper::ActiveJobSample1Wrapper" -> "Job3";
66
71
  "Job2" -> "Job4";
67
72
  "Job3" -> "Job4";
68
73
  "Job3" -> "InnerJob3";
data/sample/result.png CHANGED
Binary file
data/sample/rukawa.rb CHANGED
@@ -2,3 +2,11 @@ Rukawa.configure do |c|
2
2
  c.graph.concentrate = true
3
3
  c.graph.nodesep = 0.8
4
4
  end
5
+
6
+ redis_host = ENV["REDIS_HOST"] || "localhost:6379"
7
+ Rukawa.configure do |c|
8
+ c.status_store = ActiveSupport::Cache::RedisStore.new(redis_host)
9
+ end
10
+
11
+ require 'active_job'
12
+ ActiveJob::Base.queue_adapter = :sucker_punch
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rukawa
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.0
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - joker1007
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-07-29 00:00:00.000000000 Z
11
+ date: 2023-04-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -16,14 +16,28 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '4'
19
+ version: '6'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '4'
26
+ version: '6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: redis
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: concurrent-ruby
29
43
  requirement: !ruby/object:Gem::Requirement
@@ -86,28 +100,28 @@ dependencies:
86
100
  requirements:
87
101
  - - "~>"
88
102
  - !ruby/object:Gem::Version
89
- version: '1.11'
103
+ version: '2.0'
90
104
  type: :development
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
108
  - - "~>"
95
109
  - !ruby/object:Gem::Version
96
- version: '1.11'
110
+ version: '2.0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: rake
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
115
  - - "~>"
102
116
  - !ruby/object:Gem::Version
103
- version: '10.0'
117
+ version: '13.0'
104
118
  type: :development
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
122
  - - "~>"
109
123
  - !ruby/object:Gem::Version
110
- version: '10.0'
124
+ version: '13.0'
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: rspec
113
127
  requirement: !ruby/object:Gem::Requirement
@@ -150,6 +164,62 @@ dependencies:
150
164
  - - ">="
151
165
  - !ruby/object:Gem::Version
152
166
  version: '0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: activejob
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - ">="
172
+ - !ruby/object:Gem::Version
173
+ version: '0'
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - ">="
179
+ - !ruby/object:Gem::Version
180
+ version: '0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: sucker_punch
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - ">="
186
+ - !ruby/object:Gem::Version
187
+ version: '0'
188
+ type: :development
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: '0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: aws-sdk
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - "~>"
200
+ - !ruby/object:Gem::Version
201
+ version: '2.0'
202
+ type: :development
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - "~>"
207
+ - !ruby/object:Gem::Version
208
+ version: '2.0'
209
+ - !ruby/object:Gem::Dependency
210
+ name: google-api-client
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: '0.9'
216
+ type: :development
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: '0.9'
153
223
  description: Hyper simple job workflow engine
154
224
  email:
155
225
  - kakyoin.hierophant@gmail.com
@@ -158,10 +228,11 @@ executables:
158
228
  extensions: []
159
229
  extra_rdoc_files: []
160
230
  files:
231
+ - ".github/workflows/test.yml"
161
232
  - ".gitignore"
162
233
  - ".rspec"
163
- - ".travis.yml"
164
234
  - Gemfile
235
+ - LICENSE.txt
165
236
  - README.md
166
237
  - Rakefile
167
238
  - bin/console
@@ -169,6 +240,10 @@ files:
169
240
  - exe/rukawa
170
241
  - lib/rukawa.rb
171
242
  - lib/rukawa/abstract_job.rb
243
+ - lib/rukawa/builtins/base.rb
244
+ - lib/rukawa/builtins/embulk.rb
245
+ - lib/rukawa/builtins/shell.rb
246
+ - lib/rukawa/builtins/waiter.rb
172
247
  - lib/rukawa/cli.rb
173
248
  - lib/rukawa/configuration.rb
174
249
  - lib/rukawa/context.rb
@@ -178,12 +253,17 @@ files:
178
253
  - lib/rukawa/job.rb
179
254
  - lib/rukawa/job_net.rb
180
255
  - lib/rukawa/overview.rb
256
+ - lib/rukawa/remote.rb
257
+ - lib/rukawa/remote/status_store.rb
181
258
  - lib/rukawa/runner.rb
182
259
  - lib/rukawa/state.rb
183
260
  - lib/rukawa/version.rb
261
+ - lib/rukawa/wrapper.rb
262
+ - lib/rukawa/wrapper/active_job.rb
184
263
  - rukawa.gemspec
185
264
  - sample/job_nets/sample_job_net.rb
186
265
  - sample/jobnet.png
266
+ - sample/jobs/active_job.rb
187
267
  - sample/jobs/sample_job.rb
188
268
  - sample/result.dot
189
269
  - sample/result.png
@@ -193,7 +273,7 @@ files:
193
273
  homepage: https://github.com/joker1007/rukawa
194
274
  licenses: []
195
275
  metadata: {}
196
- post_install_message:
276
+ post_install_message:
197
277
  rdoc_options: []
198
278
  require_paths:
199
279
  - lib
@@ -208,9 +288,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
208
288
  - !ruby/object:Gem::Version
209
289
  version: '0'
210
290
  requirements: []
211
- rubyforge_project:
212
- rubygems_version: 2.6.4
213
- signing_key:
291
+ rubygems_version: 3.4.10
292
+ signing_key:
214
293
  specification_version: 4
215
294
  summary: Hyper simple job workflow engine
216
295
  test_files: []
data/.travis.yml DELETED
@@ -1,6 +0,0 @@
1
- language: ruby
2
- sudo: false
3
- rvm:
4
- - 2.2.5
5
- - 2.3.0
6
- before_install: gem install bundler -v 1.11.2