workflow_manager 0.7.8 → 0.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/job_checker.rb +17 -5
- data/lib/workflow_manager/server.rb +6 -3
- data/lib/workflow_manager/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bb29ff98b1a798d31a2a6d3bc84b0a2145d975e8b5e30ef242f309eb74edd099
|
4
|
+
data.tar.gz: 8214d4df1d712c8926e771999105201f141c2495a86008c0dd661b45f6826ad3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e9fb82f9743d5545235fab63e95c892d6acb879fd1d13c21ffba15df0cff53bc131a5b89453234e48fc13d468c24dced45f948dfe54f1c28fcb4324b5f758bc
|
7
|
+
data.tar.gz: 6754552645ac73a6609f8e5175f248892a138c0a20392a54fd79de1395c0e7c4e273c3efea1fc984867fcdd3894aa23724026d5699dbcace2bfee441eedc0f09
|
data/lib/job_checker.rb
CHANGED
@@ -43,15 +43,15 @@ class JobChecker
|
|
43
43
|
end
|
44
44
|
new_job_script
|
45
45
|
end
|
46
|
-
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host)
|
46
|
+
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
|
47
47
|
unless @start_time
|
48
48
|
@start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
49
49
|
end
|
50
50
|
time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
51
|
-
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host].join(',')
|
51
|
+
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
|
52
52
|
end
|
53
53
|
|
54
|
-
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil)
|
54
|
+
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
|
55
55
|
puts "JobID (in JobChecker): #{job_id}"
|
56
56
|
db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
|
57
57
|
db1 = Redis.new(port: PORT, db: 1) # log DB
|
@@ -66,10 +66,10 @@ class JobChecker
|
|
66
66
|
#print ret
|
67
67
|
state = ret.split(/\n/).last.strip
|
68
68
|
#puts "state: #{state}"
|
69
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
69
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
70
70
|
|
71
71
|
unless state == pre_state
|
72
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
72
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
73
73
|
project_jobs = eval((db2[project_id]||[]).to_s)
|
74
74
|
project_jobs = Hash[*project_jobs]
|
75
75
|
project_jobs[job_id] = state
|
@@ -79,10 +79,22 @@ class JobChecker
|
|
79
79
|
pre_state = state
|
80
80
|
sleep WORKER_INTERVAL
|
81
81
|
end while state =~ /RUNNING/ or state =~ /PENDING/ or state =~ /---/
|
82
|
+
|
83
|
+
# post process
|
82
84
|
if next_dataset_id and rails_host
|
83
85
|
uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
|
84
86
|
#p uri
|
85
87
|
res = Net::HTTP.get_response(uri)
|
88
|
+
|
89
|
+
if log_dir and !log_dir.empty?
|
90
|
+
copy_command = copy_command_template.gsub("org_file", log_file)
|
91
|
+
#puts "copy_command=#{copy_command}"
|
92
|
+
system copy_command
|
93
|
+
err_file = log_file.gsub('_o.log','_e.log')
|
94
|
+
copy_command = copy_command_template.gsub("org_file", err_file)
|
95
|
+
#puts "copy_command=#{copy_command}"
|
96
|
+
system copy_command
|
97
|
+
end
|
86
98
|
end
|
87
99
|
end
|
88
100
|
end
|
@@ -187,12 +187,13 @@ module WorkflowManager
|
|
187
187
|
statuses.each do |job_id, status|
|
188
188
|
# puts [job_id, status].join(",")
|
189
189
|
# 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
|
190
|
-
stat, script_basename, time, user, project_number, next_dataset_id, rails_host = status.split(",")
|
190
|
+
stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
|
191
191
|
if stat == "RUNNING" or stat == "PENDING"
|
192
192
|
log_file = logs[job_id]
|
193
193
|
log_puts("JobID (in recovery check): #{job_id}")
|
194
194
|
puts "JobID (in recovery check): #{job_id}"
|
195
|
-
|
195
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
196
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
196
197
|
end
|
197
198
|
end
|
198
199
|
end
|
@@ -304,8 +305,10 @@ module WorkflowManager
|
|
304
305
|
#p log_file
|
305
306
|
#p job_id
|
306
307
|
puts "JobID (in WorkflowManager): #{job_id}"
|
308
|
+
#puts "log_dir=#{log_dir}"
|
307
309
|
sleep 1
|
308
|
-
|
310
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
311
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
309
312
|
job_id
|
310
313
|
end
|
311
314
|
def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workflow_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|