workflow_manager 0.0.5

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in workflow_manager.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Function Genomics Center Zurich
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,29 @@
1
+ # WorkflowManager
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'workflow_manager'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install workflow_manager
18
+
19
+ ## Usage
20
+
21
+ TODO: Write usage instructions here
22
+
23
+ ## Contributing
24
+
25
+ 1. Fork it
26
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
27
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
28
+ 4. Push to the branch (`git push origin my-new-feature`)
29
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
data/bin/wfm_get_log ADDED
@@ -0,0 +1,32 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+ # 20121112 masa workflow manager client
4
+ Version = '20130307-094928'
5
+
6
+ require 'drb/drb'
7
+
8
+ unless job_id = ARGV[0]
9
+ puts "Usage:\n #{__FILE__} [job_id] [with_err] [server]"
10
+ puts
11
+ puts " job_id: required"
12
+ puts " with_err: if you want also standard errors"
13
+ puts " server: workflow_manager sever URI (default: druby://localhost:12345)"
14
+ exit
15
+ end
16
+ with_err = ARGV[1]
17
+ uri = ARGV[2]||'druby://localhost:12345'
18
+
19
+ wfmrc = if File.exist?(".wfmrc")
20
+ ".wfmrc"
21
+ elsif File.exist?(File.expand_path("~/.wfmrc"))
22
+ File.expand_path("~/.wfmrc")
23
+ end
24
+ if wfmrc
25
+ File.readlines(wfmrc).each do |line|
26
+ if line =~ /server:\s*(druby.+)/
27
+ uri = $1
28
+ end
29
+ end
30
+ end
31
+ workflow_manager = DRbObject.new_with_uri(uri)
32
+ puts workflow_manager.get_log(job_id, with_err)
@@ -0,0 +1,29 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+ # 20121112 masa workflow manager client
4
+ Version = '20130307-094955'
5
+
6
+ require 'drb/drb'
7
+
8
+ unless job_id = ARGV[0]
9
+ puts "Usage:\n #{__FILE__} [job_id] [server]"
10
+ puts
11
+ puts " job_id: required"
12
+ puts " server: workflow_manager URI (default: druby://localhost:12345)"
13
+ exit
14
+ end
15
+ uri = ARGV[1]||'druby://localhost:12345'
16
+ wfmrc = if File.exist?(".wfmrc")
17
+ ".wfmrc"
18
+ elsif File.exist?(File.expand_path("~/.wfmrc"))
19
+ File.expand_path("~/.wfmrc")
20
+ end
21
+ if wfmrc
22
+ File.readlines(wfmrc).each do |line|
23
+ if line =~ /server:\s*(druby.+)/
24
+ uri = $1
25
+ end
26
+ end
27
+ end
28
+ workflow_manager = DRbObject.new_with_uri(uri)
29
+ puts workflow_manager.get_script(job_id)
data/bin/wfm_hello ADDED
@@ -0,0 +1,30 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+ # 20121112 masa workflow manager client
4
+ Version = '20130517-111334'
5
+
6
+ require 'drb/drb'
7
+
8
+ if ARGV[0]=='-h' or ARGV[0]=='--help'
9
+ puts "Usage:\n #{__FILE__}"
10
+ exit
11
+ end
12
+
13
+ uri = ARGV[0]||'druby://localhost:12345'
14
+ wfmrc = if File.exist?(".wfmrc")
15
+ ".wfmrc"
16
+ elsif File.exist?(File.expand_path("~/.wfmrc"))
17
+ File.expand_path("~/.wfmrc")
18
+ end
19
+ if wfmrc
20
+ File.readlines(wfmrc).each do |line|
21
+ if line =~ /server:\s*(druby.+)/
22
+ uri = $1
23
+ end
24
+ if line =~ /project:\s*(\d+)/
25
+ project_number = $1
26
+ end
27
+ end
28
+ end
29
+ workflow_manager = DRbObject.new_with_uri(uri)
30
+ puts workflow_manager.hello
data/bin/wfm_job_list ADDED
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+ # 20121112 masa workflow manager client
4
+ Version = '20131104-192211'
5
+
6
+ require 'drb/drb'
7
+ require 'workflow_manager/optparse_ex'
8
+ #require File.join((File.expand_path('../../lib',__FILE__)), 'optparse_ex.rb')
9
+
10
+ opt = OptionParser.new do |o|
11
+ o.banner = "Usage:\n #{File.basename(__FILE__)} [options]"
12
+ # o.on(:user, 'sushi lover', '-u user', '--user', 'who submitted?')
13
+ o.on(:project, nil, '-p project', '--project', 'project number')
14
+ o.on(:server, 'druby://localhost:12345', '-d server', '--server', 'workflow manager URI (default: druby://localhost:12345)')
15
+ o.parse!(ARGV)
16
+ end
17
+
18
+ with_result = false
19
+ uri = opt.server||'druby://localhost:12345'
20
+ project_number = opt.project
21
+ wfmrc = if File.exist?(".wfmrc")
22
+ ".wfmrc"
23
+ elsif File.exist?(File.expand_path("~/.wfmrc"))
24
+ File.expand_path("~/.wfmrc")
25
+ end
26
+ if wfmrc
27
+ File.readlines(wfmrc).each do |line|
28
+ if line =~ /server:\s*(druby.+)/
29
+ uri = $1
30
+ end
31
+ if line =~ /project:\s*(\d+)/
32
+ project_number = $1
33
+ end
34
+ end
35
+ end
36
+ workflow_manager = DRbObject.new_with_uri(uri)
37
+ puts workflow_manager.job_list(with_result, project_number)
@@ -0,0 +1,55 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+ # 20121112 masa workflow manager client
4
+ Version = '20131104-192323'
5
+
6
+ require 'drb/drb'
7
+ require 'workflow_manager/optparse_ex'
8
+
9
+ opt = OptionParser.new do |o|
10
+ o.banner = "Usage:\n #{File.basename(__FILE__)} [options] [job_script.sh]"
11
+ o.on(:user, 'sushi lover', '-u user', '--user', 'who submitted? (default: sushi lover)')
12
+ o.on(:project, 1001, '-p project', '--project', 'project number (default: 1001)')
13
+ o.on(:server, 'druby://localhost:12345', '-d server', '--server', 'workflow manager URI (default: druby://localhost:12345)')
14
+ o.on(:log, '-o logdir', '--logdir', 'directory of standard output and standard error file outputted after the job')
15
+ o.on(:cores, '-c cores', '--cores', 'Number of cores to request for g-sub')
16
+ o.on(:nodes, '-n nodes', '--nodes', 'Comma separated list of nodes to submit to for g-sub')
17
+ o.on(:ram, '-r RAM', '--RAM', 'Amount of RAM to request in Gigs for g-sub')
18
+ o.on(:scratch, '-s scratch', '--scratch', 'Amount of scratch space to request in Gigs for g-sub')
19
+ o.parse!(ARGV)
20
+ end
21
+ unless script_file = ARGV[0] and script_file =~ /\.sh/
22
+ print opt.help
23
+ exit
24
+ end
25
+
26
+ project_number = opt.project
27
+ uri = opt.server
28
+ user = opt.user
29
+ wfmrc = if File.exist?(".wfmrc")
30
+ ".wfmrc"
31
+ elsif File.exist?(File.expand_path("~/.wfmrc"))
32
+ File.expand_path("~/.wfmrc")
33
+ end
34
+ if wfmrc
35
+ File.readlines(wfmrc).each do |line|
36
+ if line =~ /user:\s*(.+)/
37
+ user = $1
38
+ end
39
+ if line =~ /server:\s*(druby.+)/
40
+ uri = $1
41
+ end
42
+ if line =~ /project:\s*(\d+)/
43
+ project_number = $1
44
+ end
45
+ end
46
+ end
47
+ sge_options = []
48
+ sge_options << "-c #{opt.cores}" if opt.cores
49
+ sge_options << "-r #{opt.ram}" if opt.ram
50
+ sge_options << "-s #{opt.scratch}" if opt.scratch
51
+ sge_options << "-n #{opt.nodes}" if opt.nodes
52
+
53
+ script_content = File.read(script_file)
54
+ workflow_manager = DRbObject.new_with_uri(uri)
55
+ puts workflow_manager.start_monitoring(script_file, user, 0, script_content, project_number, sge_options.join(' '), opt.log)
data/bin/wfm_status ADDED
@@ -0,0 +1,29 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+ # 20121112 masa workflow manager client
4
+ Version = '20130307-095018'
5
+
6
+ require 'drb/drb'
7
+
8
+ unless job_id = ARGV[0]
9
+ puts "Usage:\n #{__FILE__} [job_id] [server]"
10
+ puts
11
+ puts " job_id: required"
12
+ puts " server: workflow_manager URI (default: druby://localhost:12345)"
13
+ exit
14
+ end
15
+ uri = ARGV[1]||'druby://localhost:12345'
16
+ wfmrc = if File.exist?(".wfmrc")
17
+ ".wfmrc"
18
+ elsif File.exist?(File.expand_path("~/.wfmrc"))
19
+ File.expand_path("~/.wfmrc")
20
+ end
21
+ if wfmrc
22
+ File.readlines(wfmrc).each do |line|
23
+ if line =~ /server:\s*(druby.+)/
24
+ uri = $1
25
+ end
26
+ end
27
+ end
28
+ workflow_manager = DRbObject.new_with_uri(uri)
29
+ puts workflow_manager.status(job_id)
@@ -0,0 +1,262 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+ # 20121109 masa workflow manager druby server
4
+ Version = '20131104-192005'
5
+
6
+ require 'drb/drb'
7
+ require 'fileutils'
8
+ require 'kyotocabinet'
9
+ require_relative '../lib/workflow_manager'
10
+
11
+
12
+ # default parameters
13
+ #LOG_DIR = '/srv/GT/analysis/workflow_manager_logs'
14
+ LOG_DIR = 'logs'
15
+ DB_DIR = 'dbs'
16
+ INTERVAL = 30
17
+ RESUBMIT = 0
18
+
19
+ class WorkflowManager
20
+ @@config = nil
21
+ class Config
22
+ attr_accessor :log_dir
23
+ attr_accessor :db_dir
24
+ attr_accessor :interval
25
+ attr_accessor :resubmit
26
+ attr_accessor :cluster
27
+ end
28
+ def self.config=(config)
29
+ @@config = config
30
+ end
31
+ def self.config
32
+ @@config
33
+ end
34
+ def config
35
+ @@config ||= WorkflowManager.configure{}
36
+ end
37
+ def self.configure
38
+ @@config = Config.new
39
+ # default values
40
+ @@config.log_dir = LOG_DIR
41
+ @@config.db_dir = DB_DIR
42
+ @@config.interval = INTERVAL # interval to check jobs, [s]
43
+ @@config.resubmit = RESUBMIT # how many times at maximum to resubmit when job fails
44
+ yield(@@config)
45
+ if @@config.cluster
46
+ @@config.cluster.log_dir = File.expand_path(@@config.log_dir)
47
+ end
48
+ @@config
49
+ end
50
+ end
51
+
52
+ class WorkflowManager
53
+ def initialize
54
+ @interval = config.interval
55
+ @resubmit = config.resubmit
56
+ @db_stat = File.join(config.db_dir, 'statuses.kch')
57
+ @db_logs = File.join(config.db_dir, 'logs.kch')
58
+
59
+ @log_dir = File.expand_path(config.log_dir)
60
+ @db_dir = File.expand_path(config.db_dir)
61
+ FileUtils.mkdir_p @log_dir unless File.exist?(@log_dir)
62
+ FileUtils.mkdir_p @db_dir unless File.exist?(@db_dir)
63
+ @statuses = KyotoCabinet::DB.new
64
+ @logs = KyotoCabinet::DB.new
65
+ @system_log = File.join(@log_dir, "system.log")
66
+ @mutex = Mutex.new
67
+ @cluster = config.cluster
68
+ log_puts("Server starts")
69
+ end
70
+ def hello
71
+ 'hello, '+ @cluster.name
72
+ end
73
+ def copy_commands(org_dir, dest_parent_dir)
74
+ @cluster.copy_commands(org_dir, dest_parent_dir)
75
+ end
76
+ def log_puts(str)
77
+ time = Time.now.strftime("[%Y.%m.%d %H:%M:%S]")
78
+ @mutex.synchronize do
79
+ open(@system_log, "a") do |out|
80
+ out.print time + " " + str + "\n"
81
+ end
82
+ end
83
+ end
84
+ def start_monitoring(submit_command, user = 'sushi lover', resubmit = 0, script = '', project_number = 0, sge_options='', log_dir = '')
85
+ log_puts("monitoring: script=" + submit_command + " user=" + user + " resubmit=" + resubmit.to_s + " project=" + project_number.to_s + " sge option=" + sge_options + " log dir=" + log_dir.to_s)
86
+
87
+ #warn submit_command
88
+ #
89
+ # TODO: analyze arguments
90
+ #
91
+ job_id, log_file, command = @cluster.submit_job(submit_command, script, sge_options)
92
+ log_puts("submit: " + job_id + " " + command)
93
+
94
+ #
95
+ # monitor worker
96
+ #
97
+ if job_id and log_file
98
+ monitor_worker = Thread.new(job_id, log_file, submit_command, user, resubmit, script, project_number, sge_options, log_dir) do |t_job_id, t_log_file, t_submit_command, t_user, t_resubmit, t_script, t_project_number, t_sge_options, t_log_dir|
99
+ loop do
100
+ status = success_or_fail(t_job_id, t_log_file)
101
+ script_name = File.basename(submit_command).split(/-/).first
102
+ @statuses.open(@db_stat)
103
+ start_time = if stat = @statuses[t_job_id] and stat = stat.split(/,/) and time = stat[2]
104
+ time
105
+ end
106
+ time = if start_time
107
+ if status == 'success' or status == 'fail'
108
+ start_time + '/' + Time.now.strftime("%Y-%m-%d %H:%M:%S")
109
+ else
110
+ start_time
111
+ end
112
+ else
113
+ Time.now.strftime("%Y-%m-%d %H:%M:%S")
114
+ end
115
+ @statuses[t_job_id] = [status, script_name, time, user, project_number].join(',')
116
+ @statuses.close
117
+ @logs.open(@db_logs)
118
+ @logs[t_job_id] = t_log_file
119
+ @logs.close
120
+ #warn t_job_id + " " + status
121
+ if status == 'success'
122
+ log_puts(status + ": " + t_job_id)
123
+ unless t_log_dir.empty?
124
+ copy_commands(t_log_file, t_log_dir).each do |command|
125
+ log_puts(command)
126
+ system command
127
+ end
128
+ err_file = t_log_file.gsub('_o.log','_e.log')
129
+ copy_commands(err_file, t_log_dir).each do |command|
130
+ log_puts(command)
131
+ system command
132
+ end
133
+ end
134
+ Thread.current.kill
135
+ elsif status == 'fail'
136
+ log_puts(status + ": " + t_job_id)
137
+ #
138
+ # TODO: re-submit
139
+ #
140
+ if t_resubmit < RESUBMIT
141
+ log_puts("resubmit: " + t_job_id)
142
+ resubmit_job_id = start_monitoring(t_submit_command, t_user, t_resubmit + 1, t_script, t_project_number, t_sge_options)
143
+ script_name = File.basename(submit_command).split(/-/).first
144
+ @statuses.open(@db_stat)
145
+ @statuses[t_job_id] = ["resubmit: " + resubmit_job_id.to_s, script_name, Time.now.strftime("%Y-%m-%d %H:%M:%S"), t_user, t_project_number].join(',')
146
+ @statuses.close
147
+ else
148
+ log_puts("fail: " + t_job_id)
149
+ end
150
+ unless t_log_dir.empty?
151
+ copy_commands(t_log_file, t_log_dir).each do |command|
152
+ log_puts(command)
153
+ system command
154
+ end
155
+ err_file = t_log_file.gsub('_o.log','_e.log')
156
+ copy_commands(err_file, t_log_dir).each do |command|
157
+ log_puts(command)
158
+ system command
159
+ end
160
+ end
161
+ Thread.current.kill
162
+ end
163
+ sleep @interval
164
+ end
165
+ end
166
+ job_id.to_i
167
+ end
168
+ end
169
+ def status(job_id)
170
+ stat = nil
171
+ @statuses.open(@db_stat)
172
+ stat = @statuses[job_id.to_s]
173
+ @statuses.close
174
+ stat
175
+ end
176
+ def job_list(with_results=false, project_number=nil)
177
+ s = []
178
+ @statuses.open(@db_stat)
179
+ @statuses.each do |key, value|
180
+ if project_number
181
+ if x = value.split(/,/)[4].to_i==project_number.to_i
182
+ s << [key, value]
183
+ end
184
+ else
185
+ s << [key, value]
186
+ end
187
+ end
188
+ @statuses.close
189
+ s.sort.reverse.map{|v| v.join(',')}.join("\n")
190
+ end
191
+ def get_log(job_id, with_err=false)
192
+ @logs.open(@db_logs)
193
+ log_file = @logs[job_id.to_s]
194
+ @logs.close
195
+ log_data = if log_file and File.exist?(log_file)
196
+ "__STDOUT LOG__\n\n" + File.read(log_file)
197
+ else
198
+ 'no log file'
199
+ end
200
+ if with_err
201
+ err_file = log_file.gsub(/_o\.log/,'_e.log')
202
+ if err_file and File.exist?(err_file)
203
+ log_data << "\n\n__STDERR LOG__\n\n"
204
+ log_data << File.read(err_file)
205
+ end
206
+ end
207
+ log_data
208
+ end
209
+ def get_script(job_id)
210
+ @logs.open(@db_logs)
211
+ script_file = @logs[job_id.to_s]
212
+ @logs.close
213
+ if script_file
214
+ script_file = script_file.gsub(/_o\.log/,'')
215
+ end
216
+ script = if script_file and File.exist?(script_file)
217
+ File.read(script_file)
218
+ else
219
+ 'no script file'
220
+ end
221
+ script
222
+ end
223
+ def success_or_fail(job_id, log_file)
224
+ job_running = @cluster.job_running?(job_id)
225
+ job_ends = @cluster.job_ends?(log_file)
226
+ msg = if job_running
227
+ 'running'
228
+ elsif job_ends
229
+ 'success'
230
+ else
231
+ 'fail'
232
+ end
233
+ msg
234
+ end
235
+ end
236
+
237
+ #
238
+ # main
239
+ #
240
+ #if __FILE__ == $0
241
+
242
+ opt = OptionParser.new do |o|
243
+ o.banner = "Usage:\n #{File.basename(__FILE__)} -d [druby://host:port] -m [development|production]"
244
+ o.on(:server, 'druby://localhost:12345', '-d server', '--server', 'workflow manager URI (default: druby://localhost:12345)')
245
+ o.on(:mode, 'development', '-m mode', '--mode', 'development|production (default: development)')
246
+ o.parse!(ARGV)
247
+ end
248
+
249
+ uri = opt.server
250
+ if opt.mode =~ /[development|production]/
251
+ config = File.join(File.dirname(File.expand_path(__FILE__)), "../config/environments/#{opt.mode}.rb")
252
+ opt.mode = nil unless File.exist?(config)
253
+ end
254
+ print "mode = #{opt.mode}\n"
255
+ if opt.mode
256
+ require_relative "../config/environments/#{opt.mode}"
257
+ end
258
+ DRb.start_service(uri, WorkflowManager.new)
259
+ puts DRb.uri
260
+ DRb.thread.join
261
+ #sleep
262
+ #end
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+
4
+ WorkflowManager.configure do |config|
5
+ config.log_dir = 'logs'
6
+ config.db_dir = 'dbs'
7
+ config.interval = 30
8
+ config.resubmit = 0
9
+ config.cluster = LocalComputer.new('local_computer')
10
+ end
11
+
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+
4
+ WorkflowManager.configure do |config|
5
+ config.log_dir = 'logs'
6
+ config.db_dir = 'dbs'
7
+ config.interval = 30
8
+ config.resubmit = 0
9
+ config.cluster = LocalComputer.new('local_computer')
10
+ end
11
+
@@ -0,0 +1,5 @@
1
+
2
+ require_relative "workflow_manager/version"
3
+
4
+ require_relative "workflow_manager/optparse_ex"
5
+ require_relative "workflow_manager/cluster"
@@ -0,0 +1,113 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+
4
+ class Cluster
5
+ attr_accessor :name
6
+ attr_reader :options
7
+ attr_accessor :log_dir
8
+ def initialize(name='', log_dir='')
9
+ @name = name
10
+ @options = {}
11
+ @log_dir = log_dir
12
+ end
13
+ def generate_new_job_script(script_name, script_content)
14
+ new_job_script = File.basename(script_name) + "_" + Time.now.strftime("%Y%m%d%H%M%S")
15
+ new_job_script = File.join(@log_dir, new_job_script)
16
+ open(new_job_script, 'w') do |out|
17
+ out.print script_content
18
+ out.print "\necho __SCRIPT END__\n"
19
+ end
20
+ new_job_script
21
+ end
22
+ def submit_job(script_file, script_content, option='')
23
+ end
24
+ def job_running?(job_id)
25
+ end
26
+ def job_ends?(log_file)
27
+ end
28
+ def copy_commands(org_dir, dest_parent_dir)
29
+ end
30
+ end
31
+
32
+ class LocalComputer < Cluster
33
+ def submit_job(script_file, script_content, option='')
34
+ if script_name = File.basename(script_file) and script_name =~ /\.sh$/
35
+ new_job_script = generate_new_job_script(script_name, script_content)
36
+ new_job_script_base = File.basename(new_job_script)
37
+ log_file = File.join(@log_dir, new_job_script_base + "_o.log")
38
+ err_file = File.join(@log_dir, new_job_script_base + "_e.log")
39
+ command = "bash #{new_job_script} 1> #{log_file} 2> #{err_file}"
40
+ pid = spawn(command)
41
+ Process.detach(pid)
42
+ [pid.to_s, log_file, command]
43
+ end
44
+ end
45
+ def job_running?(pid)
46
+ command = "ps aux"
47
+ result = IO.popen(command) do |io|
48
+ flag = false
49
+ while line=io.gets
50
+ x = line.split
51
+ if x[1].to_i == pid.to_i
52
+ flag = true
53
+ break
54
+ end
55
+ end
56
+ flag
57
+ end
58
+ result
59
+ end
60
+ def job_ends?(log_file)
61
+ command = "tail -n 20 #{log_file}|grep '__SCRIPT END__'"
62
+ result = `#{command}`
63
+ result.to_s.empty? ? false : true
64
+ end
65
+ def copy_commands(org_dir, dest_parent_dir)
66
+ commands = []
67
+ commands << "mkdir -p #{dest_parent_dir}"
68
+ commands << "cp -r #{org_dir} #{dest_parent_dir}"
69
+ commands
70
+ end
71
+ end
72
+
73
+ class FGCZCluster < Cluster
74
+ def submit_job(script_file, script_content, option='')
75
+ if script_name = File.basename(script_file) and script_name =~ /\.sh$/
76
+ new_job_script = generate_new_job_script(script_name, script_content)
77
+ new_job_script_base = File.basename(new_job_script)
78
+ log_file = File.join(@log_dir, new_job_script_base + "_o.log")
79
+ err_file = File.join(@log_dir, new_job_script_base + "_e.log")
80
+ command = "g-sub -o #{log_file} -e #{err_file} #{option} #{new_job_script}"
81
+ job_id = `#{command}`
82
+ job_id = job_id.match(/Your job (\d+) \(/)[1]
83
+ [job_id, log_file, command]
84
+ end
85
+ end
86
+ def job_running?(job_id)
87
+ qstat_flag = false
88
+ IO.popen('qstat -u "*"') do |io|
89
+ while line=io.gets
90
+ if line =~ /#{job_id}/
91
+ qstat_flag = true
92
+ break
93
+ end
94
+ end
95
+ end
96
+ qstat_flag
97
+ end
98
+ def job_ends?(log_file)
99
+ log_flag = false
100
+ IO.popen("tail -n 10 #{log_file}") do |io|
101
+ while line=io.gets
102
+ if line =~ /__SCRIPT END__/
103
+ log_flag = true
104
+ break
105
+ end
106
+ end
107
+ end
108
+ log_flag
109
+ end
110
+ def copy_commands(org_dir, dest_parent_dir)
111
+ commands = ["g-req -w copy #{org_dir} #{dest_parent_dir}"]
112
+ end
113
+ end
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+
4
+ require 'optparse'
5
+ # Version = '20130517-120455'
6
+
7
+ class OptionParser
8
+ attr_reader :p
9
+ alias :_on :on
10
+ def on(attr, *args, &block)
11
+ if attr.is_a?(Symbol)
12
+ self.class.class_eval do
13
+ unless method_defined?(attr)
14
+ attr_accessor attr
15
+ else
16
+ raise "Method #{attr.to_s} is already defined in OptionParser class"
17
+ end
18
+ end
19
+ unless args[0] =~ /\-/
20
+ default = args.shift
21
+ self.send(attr.to_s+"=", default)
22
+ end
23
+ _on(*args) do |i|
24
+ self.send(attr.to_s+"=", i)
25
+ block.call(i) if block
26
+ end
27
+ else
28
+ args.unshift attr
29
+ _on(*args, block)
30
+ end
31
+ end
32
+ end
33
+
34
+ if __FILE__ == $0
35
+ opt = OptionParser.new do |o|
36
+ o.banner = "Last update: #{o.version}\nUsage: ruby #{__FILE__} [options]"
37
+ o.on(:size, 100, '-N N', '--pop_size', Integer, 'population size (default: 100)'){|i| p i}
38
+ o.on(:seed, '-R R', '--rseed', 'random seed')
39
+ o.on(:flag, '-f', 'flag')
40
+ o.on('-M M', '--hoge', Integer, 'hoge'){|i| p i}
41
+ o.parse!(ARGV)
42
+ end
43
+
44
+ print "opt.flag = "
45
+ p opt.flag
46
+ print "opt.size = "
47
+ p opt.size
48
+ print "opt.seed = "
49
+ p opt.seed
50
+ puts
51
+ puts opt.help
52
+ end
@@ -0,0 +1,3 @@
1
+ class WorkflowManager
2
+ VERSION = "0.0.5"
3
+ end
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'workflow_manager/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "workflow_manager"
8
+ spec.version = WorkflowManager::VERSION
9
+ spec.authors = ["Functional Genomics Center Zurich"]
10
+ spec.email = ["masaomi.hatakeyama@fgcz.uzh.ch"]
11
+ spec.description = %q{Workflow Manager manages job submissions using dRuby.}
12
+ spec.summary = %q{Workflow Manager manages job submissions using dRuby.}
13
+ spec.homepage = ""
14
+ spec.license = "MIT"
15
+
16
+ #spec.files = `git ls-files`.split($/)
17
+ spec.files = `bzr ls --versioned --recursive`.split($/).select{|file| !File.directory?(file)}
18
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
19
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
20
+ spec.require_paths = ["lib"]
21
+
22
+ spec.add_development_dependency "bundler", "~> 1.3"
23
+ spec.add_development_dependency "rake"
24
+ end
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: workflow_manager
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.5
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Functional Genomics Center Zurich
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-11-07 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: '1.3'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: '1.3'
30
+ - !ruby/object:Gem::Dependency
31
+ name: rake
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ description: Workflow Manager manages job submissions using dRuby.
47
+ email:
48
+ - masaomi.hatakeyama@fgcz.uzh.ch
49
+ executables:
50
+ - wfm_get_log
51
+ - wfm_get_script
52
+ - wfm_hello
53
+ - wfm_job_list
54
+ - wfm_monitoring
55
+ - wfm_status
56
+ - workflow_manager
57
+ extensions: []
58
+ extra_rdoc_files: []
59
+ files:
60
+ - Gemfile
61
+ - LICENSE.txt
62
+ - README.md
63
+ - Rakefile
64
+ - bin/wfm_get_log
65
+ - bin/wfm_get_script
66
+ - bin/wfm_hello
67
+ - bin/wfm_job_list
68
+ - bin/wfm_monitoring
69
+ - bin/wfm_status
70
+ - bin/workflow_manager
71
+ - config/environments/development.rb
72
+ - config/environments/production.rb
73
+ - lib/workflow_manager/cluster.rb
74
+ - lib/workflow_manager/optparse_ex.rb
75
+ - lib/workflow_manager/version.rb
76
+ - lib/workflow_manager.rb
77
+ - workflow_manager.gemspec
78
+ homepage: ''
79
+ licenses:
80
+ - MIT
81
+ post_install_message:
82
+ rdoc_options: []
83
+ require_paths:
84
+ - lib
85
+ required_ruby_version: !ruby/object:Gem::Requirement
86
+ none: false
87
+ requirements:
88
+ - - ! '>='
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ none: false
93
+ requirements:
94
+ - - ! '>='
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ requirements: []
98
+ rubyforge_project:
99
+ rubygems_version: 1.8.24
100
+ signing_key:
101
+ specification_version: 3
102
+ summary: Workflow Manager manages job submissions using dRuby.
103
+ test_files: []