workflow_manager 0.7.5 → 0.7.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/wfm_monitoring +6 -4
- data/config/environments/redis.conf +2 -2
- data/lib/job_checker.rb +25 -5
- data/lib/workflow_manager/cluster.rb +4 -0
- data/lib/workflow_manager/server.rb +21 -4
- data/lib/workflow_manager/version.rb +1 -1
- data/start_workflow_manager.sh +1 -1
- data/test/job_list.rb +11 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bb29ff98b1a798d31a2a6d3bc84b0a2145d975e8b5e30ef242f309eb74edd099
|
4
|
+
data.tar.gz: 8214d4df1d712c8926e771999105201f141c2495a86008c0dd661b45f6826ad3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e9fb82f9743d5545235fab63e95c892d6acb879fd1d13c21ffba15df0cff53bc131a5b89453234e48fc13d468c24dced45f948dfe54f1c28fcb4324b5f758bc
|
7
|
+
data.tar.gz: 6754552645ac73a6609f8e5175f248892a138c0a20392a54fd79de1395c0e7c4e273c3efea1fc984867fcdd3894aa23724026d5699dbcace2bfee441eedc0f09
|
data/bin/wfm_monitoring
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# encoding: utf-8
|
3
3
|
# 20121112 masa workflow manager client
|
4
|
-
Version = '
|
4
|
+
Version = '20211104-160323'
|
5
5
|
|
6
6
|
require 'drb/drb'
|
7
7
|
require 'workflow_manager/optparse_ex'
|
@@ -13,10 +13,11 @@ opt = OptionParser.new do |o|
|
|
13
13
|
o.on(:server, 'druby://localhost:12345', '-d server', '--server', 'workflow manager URI (default: druby://localhost:12345)')
|
14
14
|
o.on(:log, '-o logdir', '--logdir', 'directory of standard output and standard error file outputted after the job')
|
15
15
|
o.on(:cores, '-c cores', '--cores', 'Number of cores to request for g-sub')
|
16
|
-
o.on(:nodes, '-n nodes', '--nodes', 'Comma separated list of nodes to submit to for g-sub')
|
16
|
+
# o.on(:nodes, '-n nodes', '--nodes', 'Comma separated list of nodes to submit to for g-sub')
|
17
17
|
o.on(:ram, '-r RAM', '--RAM', 'Amount of RAM to request in Gigs for g-sub')
|
18
18
|
o.on(:scratch, '-s scratch', '--scratch', 'Amount of scratch space to request in Gigs for g-sub')
|
19
|
-
o.on(:queue, '-q queue', '--queue', 'Queue name')
|
19
|
+
# o.on(:queue, '-q queue', '--queue', 'Queue name')
|
20
|
+
o.on(:nice, '-i nice', '--nice', 'Nice')
|
20
21
|
o.parse!(ARGV)
|
21
22
|
end
|
22
23
|
unless script_file = ARGV[0] and script_file =~ /\.sh/
|
@@ -49,7 +50,8 @@ sge_options = []
|
|
49
50
|
sge_options << "-c #{opt.cores}" if opt.cores
|
50
51
|
sge_options << "-r #{opt.ram}" if opt.ram
|
51
52
|
sge_options << "-s #{opt.scratch}" if opt.scratch
|
52
|
-
sge_options << "-n #{opt.nodes}" if opt.nodes
|
53
|
+
#sge_options << "-n #{opt.nodes}" if opt.nodes
|
54
|
+
sge_options << "-i #{opt.nice}" if opt.nice
|
53
55
|
|
54
56
|
script_content = File.read(script_file)
|
55
57
|
workflow_manager = DRbObject.new_with_uri(uri)
|
data/lib/job_checker.rb
CHANGED
@@ -1,6 +1,9 @@
|
|
1
1
|
require 'sidekiq'
|
2
2
|
require 'redis'
|
3
3
|
|
4
|
+
require 'uri'
|
5
|
+
require 'net/http'
|
6
|
+
|
4
7
|
WORKER_INTERVAL = 10 # [s]
|
5
8
|
REDIS_CONF = File.expand_path("../../config/environments/redis.conf", __FILE__)
|
6
9
|
PORT = if File.exist?(REDIS_CONF)
|
@@ -40,15 +43,15 @@ class JobChecker
|
|
40
43
|
end
|
41
44
|
new_job_script
|
42
45
|
end
|
43
|
-
def update_time_status(status, script_basename, user, project_number, next_dataset_id)
|
46
|
+
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
|
44
47
|
unless @start_time
|
45
48
|
@start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
46
49
|
end
|
47
50
|
time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
48
|
-
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id].join(',')
|
51
|
+
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
|
49
52
|
end
|
50
53
|
|
51
|
-
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil)
|
54
|
+
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
|
52
55
|
puts "JobID (in JobChecker): #{job_id}"
|
53
56
|
db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
|
54
57
|
db1 = Redis.new(port: PORT, db: 1) # log DB
|
@@ -63,10 +66,10 @@ class JobChecker
|
|
63
66
|
#print ret
|
64
67
|
state = ret.split(/\n/).last.strip
|
65
68
|
#puts "state: #{state}"
|
66
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
|
69
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
67
70
|
|
68
71
|
unless state == pre_state
|
69
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
|
72
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
70
73
|
project_jobs = eval((db2[project_id]||[]).to_s)
|
71
74
|
project_jobs = Hash[*project_jobs]
|
72
75
|
project_jobs[job_id] = state
|
@@ -76,6 +79,23 @@ class JobChecker
|
|
76
79
|
pre_state = state
|
77
80
|
sleep WORKER_INTERVAL
|
78
81
|
end while state =~ /RUNNING/ or state =~ /PENDING/ or state =~ /---/
|
82
|
+
|
83
|
+
# post process
|
84
|
+
if next_dataset_id and rails_host
|
85
|
+
uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
|
86
|
+
#p uri
|
87
|
+
res = Net::HTTP.get_response(uri)
|
88
|
+
|
89
|
+
if log_dir and !log_dir.empty?
|
90
|
+
copy_command = copy_command_template.gsub("org_file", log_file)
|
91
|
+
#puts "copy_command=#{copy_command}"
|
92
|
+
system copy_command
|
93
|
+
err_file = log_file.gsub('_o.log','_e.log')
|
94
|
+
copy_command = copy_command_template.gsub("org_file", err_file)
|
95
|
+
#puts "copy_command=#{copy_command}"
|
96
|
+
system copy_command
|
97
|
+
end
|
98
|
+
end
|
79
99
|
end
|
80
100
|
end
|
81
101
|
|
@@ -450,11 +450,15 @@ module WorkflowManager
|
|
450
450
|
partition = if i = options.index("-p")
|
451
451
|
options[i+1]
|
452
452
|
end
|
453
|
+
nice = if i = options.index("-i")
|
454
|
+
options[i+1]
|
455
|
+
end
|
453
456
|
new_options = []
|
454
457
|
new_options << "--mem=#{ram}G" if ram
|
455
458
|
new_options << "-n #{cores}" if cores
|
456
459
|
new_options << "--tmp=#{scratch}G" if scratch
|
457
460
|
new_options << "-p #{partition}" if partition
|
461
|
+
new_options << "--nice=#{nice}" if nice
|
458
462
|
new_options.join(" ")
|
459
463
|
end
|
460
464
|
def submit_job(script_file, script_content, option='')
|
@@ -163,6 +163,7 @@ module WorkflowManager
|
|
163
163
|
RedisDB.new(1, @redis_conf)
|
164
164
|
end
|
165
165
|
@jobs = RedisDB.new(2, @redis_conf)
|
166
|
+
@trees = RedisDB.new(4, @redis_conf)
|
166
167
|
|
167
168
|
@system_log = File.join(@log_dir, "system.log")
|
168
169
|
@mutex = Mutex.new
|
@@ -186,12 +187,13 @@ module WorkflowManager
|
|
186
187
|
statuses.each do |job_id, status|
|
187
188
|
# puts [job_id, status].join(",")
|
188
189
|
# 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
|
189
|
-
stat, script_basename, time, user, project_number, next_dataset_id = status.split(",")
|
190
|
+
stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
|
190
191
|
if stat == "RUNNING" or stat == "PENDING"
|
191
192
|
log_file = logs[job_id]
|
192
193
|
log_puts("JobID (in recovery check): #{job_id}")
|
193
194
|
puts "JobID (in recovery check): #{job_id}"
|
194
|
-
|
195
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
196
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
195
197
|
end
|
196
198
|
end
|
197
199
|
end
|
@@ -296,15 +298,17 @@ module WorkflowManager
|
|
296
298
|
Thread.current.kill
|
297
299
|
end
|
298
300
|
end
|
299
|
-
def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='', next_dataset_id='')
|
301
|
+
def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='', next_dataset_id='', rails_host=nil)
|
300
302
|
script_basename = File.basename(script_path)
|
301
303
|
job_id, log_file, command = @cluster.submit_job(script_path, script_content, sge_options)
|
302
304
|
#p command
|
303
305
|
#p log_file
|
304
306
|
#p job_id
|
305
307
|
puts "JobID (in WorkflowManager): #{job_id}"
|
308
|
+
#puts "log_dir=#{log_dir}"
|
306
309
|
sleep 1
|
307
|
-
|
310
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
311
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
308
312
|
job_id
|
309
313
|
end
|
310
314
|
def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
|
@@ -555,6 +559,19 @@ module WorkflowManager
|
|
555
559
|
def cluster_node_list
|
556
560
|
@cluster.node_list
|
557
561
|
end
|
562
|
+
def save_dataset_tree(project_number, json)
|
563
|
+
@trees.transaction do |trees|
|
564
|
+
trees[project_number] = json
|
565
|
+
end
|
566
|
+
json
|
567
|
+
end
|
568
|
+
def load_dataset_tree(project_number)
|
569
|
+
json = nil
|
570
|
+
@trees.transaction do |trees|
|
571
|
+
json = trees[project_number]
|
572
|
+
end
|
573
|
+
json
|
574
|
+
end
|
558
575
|
end
|
559
576
|
end
|
560
577
|
|
data/start_workflow_manager.sh
CHANGED
data/test/job_list.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# encoding: utf-8
|
3
|
-
# Version = '
|
3
|
+
# Version = '20211001-104513'
|
4
4
|
|
5
5
|
PORT = (ARGV[0]||6380).to_i
|
6
6
|
require 'redis'
|
7
7
|
db0 = Redis.new(port: PORT, db: 0)
|
8
8
|
db1 = Redis.new(port: PORT, db: 1)
|
9
9
|
db2 = Redis.new(port: PORT, db: 2)
|
10
|
-
|
10
|
+
db4 = Redis.new(port: PORT, db: 4)
|
11
11
|
|
12
12
|
class Redis
|
13
13
|
def show_all
|
@@ -18,8 +18,8 @@ class Redis
|
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
|
-
dbs = [db0, db1, db2]
|
22
|
-
db_notes = ["state DB", "log DB", "project job DB"]
|
21
|
+
dbs = [db0, db1, db2, db4]
|
22
|
+
db_notes = ["state DB", "log DB", "project job DB", "JS tree DB"]
|
23
23
|
|
24
24
|
dbs.each.with_index do |db, i|
|
25
25
|
note = db_notes[i]
|
@@ -48,3 +48,10 @@ db2.keys.sort.each do |key|
|
|
48
48
|
value = db2.get(key)
|
49
49
|
puts [key, value].join("\t")
|
50
50
|
end
|
51
|
+
|
52
|
+
puts
|
53
|
+
puts "db3, status DB3, project specific"
|
54
|
+
db3.keys.sort.each do |key|
|
55
|
+
value = db3.get(key)
|
56
|
+
puts [key, value].join("\t")
|
57
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workflow_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|