workflow_manager 0.7.4 → 0.7.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/job_checker.rb +5 -5
- data/lib/workflow_manager/server.rb +16 -10
- data/lib/workflow_manager/version.rb +1 -1
- data/start_workflow_manager.sh +11 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5a4bcfadba28876bebd8d498893f8b47aac25ac94ec804c7c0a36df737d53db0
|
4
|
+
data.tar.gz: a41195ba36ffc6d28899254d69f89ee15f631c88d4d0d35a94b604f5d922d4cd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c90748c7b70486e627a207ef85ffea4f3f1ad965c2083b15225a86e8731ec6248a2e35512757859f9696d4eb6f2d5dd8aea873f574734f3686e2692fb6e6e31b
|
7
|
+
data.tar.gz: c425daa92e3d7bb94a4d6a3eb30dcbff2e38e5cd0d0d4c4e556beda69d094f94ae9ac6fb34523400ea07ec7c29070ed729dac1e808c50ffd1b71cf46844263ea
|
data/lib/job_checker.rb
CHANGED
@@ -40,15 +40,15 @@ class JobChecker
|
|
40
40
|
end
|
41
41
|
new_job_script
|
42
42
|
end
|
43
|
-
def update_time_status(status, script_basename, user, project_number)
|
43
|
+
def update_time_status(status, script_basename, user, project_number, next_dataset_id)
|
44
44
|
unless @start_time
|
45
45
|
@start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
46
46
|
end
|
47
47
|
time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
48
|
-
[status, script_basename, [@start_time, time].join("/"), user, project_number].join(',')
|
48
|
+
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id].join(',')
|
49
49
|
end
|
50
50
|
|
51
|
-
def perform(job_id, script_basename, log_file, user, project_id)
|
51
|
+
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil)
|
52
52
|
puts "JobID (in JobChecker): #{job_id}"
|
53
53
|
db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
|
54
54
|
db1 = Redis.new(port: PORT, db: 1) # log DB
|
@@ -63,10 +63,10 @@ class JobChecker
|
|
63
63
|
#print ret
|
64
64
|
state = ret.split(/\n/).last.strip
|
65
65
|
#puts "state: #{state}"
|
66
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id)
|
66
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
|
67
67
|
|
68
68
|
unless state == pre_state
|
69
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id)
|
69
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
|
70
70
|
project_jobs = eval((db2[project_id]||[]).to_s)
|
71
71
|
project_jobs = Hash[*project_jobs]
|
72
72
|
project_jobs[job_id] = state
|
@@ -186,12 +186,12 @@ module WorkflowManager
|
|
186
186
|
statuses.each do |job_id, status|
|
187
187
|
# puts [job_id, status].join(",")
|
188
188
|
# 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
|
189
|
-
stat, script_basename, time, user, project_number = status.split(",")
|
189
|
+
stat, script_basename, time, user, project_number, next_dataset_id = status.split(",")
|
190
190
|
if stat == "RUNNING" or stat == "PENDING"
|
191
191
|
log_file = logs[job_id]
|
192
192
|
log_puts("JobID (in recovery check): #{job_id}")
|
193
193
|
puts "JobID (in recovery check): #{job_id}"
|
194
|
-
JobChecker.perform_async(job_id, script_basename, log_file, user, project_number)
|
194
|
+
JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id)
|
195
195
|
end
|
196
196
|
end
|
197
197
|
end
|
@@ -296,7 +296,7 @@ module WorkflowManager
|
|
296
296
|
Thread.current.kill
|
297
297
|
end
|
298
298
|
end
|
299
|
-
def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
|
299
|
+
def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='', next_dataset_id='')
|
300
300
|
script_basename = File.basename(script_path)
|
301
301
|
job_id, log_file, command = @cluster.submit_job(script_path, script_content, sge_options)
|
302
302
|
#p command
|
@@ -304,7 +304,7 @@ module WorkflowManager
|
|
304
304
|
#p job_id
|
305
305
|
puts "JobID (in WorkflowManager): #{job_id}"
|
306
306
|
sleep 1
|
307
|
-
JobChecker.perform_async(job_id, script_basename, log_file, user, project_number)
|
307
|
+
JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id)
|
308
308
|
job_id
|
309
309
|
end
|
310
310
|
def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
|
@@ -473,17 +473,23 @@ module WorkflowManager
|
|
473
473
|
job_idsh = if job_ids
|
474
474
|
Hash[*(job_ids.split(',')).map{|job_id| [job_id, true]}.flatten]
|
475
475
|
end
|
476
|
-
|
477
|
-
|
476
|
+
if project_number
|
477
|
+
s_ = {}
|
478
478
|
@jobs.transaction do |jobs|
|
479
479
|
if project_jobs = jobs[project_number]
|
480
480
|
s_ = Hash[*eval(project_jobs)]
|
481
481
|
end
|
482
482
|
end
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
|
483
|
+
@statuses.transaction do |statuses|
|
484
|
+
s_.each do |job_id, stat|
|
485
|
+
s << [job_id, statuses[job_id]]
|
486
|
+
end
|
487
|
+
end
|
488
|
+
else
|
489
|
+
@statuses.transaction do |statuses|
|
490
|
+
statuses.each do |key, value|
|
491
|
+
s << [key, value]
|
492
|
+
end
|
487
493
|
end
|
488
494
|
end
|
489
495
|
if job_ids
|
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/bash
|
2
|
+
source /usr/local/ngseq/etc/lmod_profile
|
3
|
+
module load Dev/Ruby/2.6.7
|
4
|
+
module load Tools/Redis/6.0.1
|
5
|
+
conda activate gtools_env
|
6
|
+
which python
|
7
|
+
which g-sub
|
8
|
+
which g-req
|
9
|
+
mkdir -p logs
|
10
|
+
mkdir -p dbs
|
11
|
+
bundle exec workflow_manager -d druby://fgcz-h-032:40001
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workflow_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-08-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -87,6 +87,7 @@ files:
|
|
87
87
|
- spec/cluster_spec.rb
|
88
88
|
- spec/server_spec.rb
|
89
89
|
- spec/spec_helper.rb
|
90
|
+
- start_workflow_manager.sh
|
90
91
|
- test/call_worker4.rb
|
91
92
|
- test/call_worker_method.rb
|
92
93
|
- test/job_list.rb
|