workflow_manager 0.7.8 → 0.7.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/job_checker.rb +17 -5
- data/lib/workflow_manager/server.rb +6 -3
- data/lib/workflow_manager/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bb29ff98b1a798d31a2a6d3bc84b0a2145d975e8b5e30ef242f309eb74edd099
|
4
|
+
data.tar.gz: 8214d4df1d712c8926e771999105201f141c2495a86008c0dd661b45f6826ad3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e9fb82f9743d5545235fab63e95c892d6acb879fd1d13c21ffba15df0cff53bc131a5b89453234e48fc13d468c24dced45f948dfe54f1c28fcb4324b5f758bc
|
7
|
+
data.tar.gz: 6754552645ac73a6609f8e5175f248892a138c0a20392a54fd79de1395c0e7c4e273c3efea1fc984867fcdd3894aa23724026d5699dbcace2bfee441eedc0f09
|
data/lib/job_checker.rb
CHANGED
@@ -43,15 +43,15 @@ class JobChecker
|
|
43
43
|
end
|
44
44
|
new_job_script
|
45
45
|
end
|
46
|
-
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host)
|
46
|
+
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
|
47
47
|
unless @start_time
|
48
48
|
@start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
49
49
|
end
|
50
50
|
time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
51
|
-
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host].join(',')
|
51
|
+
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
|
52
52
|
end
|
53
53
|
|
54
|
-
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil)
|
54
|
+
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
|
55
55
|
puts "JobID (in JobChecker): #{job_id}"
|
56
56
|
db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
|
57
57
|
db1 = Redis.new(port: PORT, db: 1) # log DB
|
@@ -66,10 +66,10 @@ class JobChecker
|
|
66
66
|
#print ret
|
67
67
|
state = ret.split(/\n/).last.strip
|
68
68
|
#puts "state: #{state}"
|
69
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
69
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
70
70
|
|
71
71
|
unless state == pre_state
|
72
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
72
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
73
73
|
project_jobs = eval((db2[project_id]||[]).to_s)
|
74
74
|
project_jobs = Hash[*project_jobs]
|
75
75
|
project_jobs[job_id] = state
|
@@ -79,10 +79,22 @@ class JobChecker
|
|
79
79
|
pre_state = state
|
80
80
|
sleep WORKER_INTERVAL
|
81
81
|
end while state =~ /RUNNING/ or state =~ /PENDING/ or state =~ /---/
|
82
|
+
|
83
|
+
# post process
|
82
84
|
if next_dataset_id and rails_host
|
83
85
|
uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
|
84
86
|
#p uri
|
85
87
|
res = Net::HTTP.get_response(uri)
|
88
|
+
|
89
|
+
if log_dir and !log_dir.empty?
|
90
|
+
copy_command = copy_command_template.gsub("org_file", log_file)
|
91
|
+
#puts "copy_command=#{copy_command}"
|
92
|
+
system copy_command
|
93
|
+
err_file = log_file.gsub('_o.log','_e.log')
|
94
|
+
copy_command = copy_command_template.gsub("org_file", err_file)
|
95
|
+
#puts "copy_command=#{copy_command}"
|
96
|
+
system copy_command
|
97
|
+
end
|
86
98
|
end
|
87
99
|
end
|
88
100
|
end
|
@@ -187,12 +187,13 @@ module WorkflowManager
|
|
187
187
|
statuses.each do |job_id, status|
|
188
188
|
# puts [job_id, status].join(",")
|
189
189
|
# 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
|
190
|
-
stat, script_basename, time, user, project_number, next_dataset_id, rails_host = status.split(",")
|
190
|
+
stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
|
191
191
|
if stat == "RUNNING" or stat == "PENDING"
|
192
192
|
log_file = logs[job_id]
|
193
193
|
log_puts("JobID (in recovery check): #{job_id}")
|
194
194
|
puts "JobID (in recovery check): #{job_id}"
|
195
|
-
|
195
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
196
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
196
197
|
end
|
197
198
|
end
|
198
199
|
end
|
@@ -304,8 +305,10 @@ module WorkflowManager
|
|
304
305
|
#p log_file
|
305
306
|
#p job_id
|
306
307
|
puts "JobID (in WorkflowManager): #{job_id}"
|
308
|
+
#puts "log_dir=#{log_dir}"
|
307
309
|
sleep 1
|
308
|
-
|
310
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
311
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
309
312
|
job_id
|
310
313
|
end
|
311
314
|
def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workflow_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|