workflow_manager 0.7.8 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/Gemfile.lock +31 -0
- data/config/environments/redis.conf +9 -5
- data/lib/job_checker.rb +17 -5
- data/lib/workflow_manager/server.rb +6 -3
- data/lib/workflow_manager/version.rb +1 -1
- data/start_workflow_manager.sh +3 -3
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5668a0ae734ac5ac90ffc891559681da2412107a29cbc8a5b279c9ac9201a27e
|
4
|
+
data.tar.gz: aa34a9a4ef006005e98d375520adfd2a8126ff835f76ff0a2d4432f439149248
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42e691408c30e9f29321199423a25fbb3844f109cf2169074262cd3a2d08bd5b9f62e1bf759e3f0448fa25fdd592606be77b7a3789cf9c1f3e3ee411122d194e
|
7
|
+
data.tar.gz: 0572fa546e578c279d6efdb8c63dac03c2075fa5bedea5c854d49a42e1c0df7e54525ab0af9ae2c54bf567f5d39b13883b1f8ab2553595872829a286546ff8ca
|
data/.gitignore
CHANGED
data/Gemfile.lock
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
workflow_manager (0.7.9)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
concurrent-ruby (1.2.0)
|
10
|
+
connection_pool (2.3.0)
|
11
|
+
rack (3.0.4.1)
|
12
|
+
redis (5.0.6)
|
13
|
+
redis-client (>= 0.9.0)
|
14
|
+
redis-client (0.12.1)
|
15
|
+
connection_pool
|
16
|
+
sidekiq (7.0.5)
|
17
|
+
concurrent-ruby (< 2)
|
18
|
+
connection_pool (>= 2.3.0)
|
19
|
+
rack (>= 2.2.4)
|
20
|
+
redis-client (>= 0.11.0)
|
21
|
+
|
22
|
+
PLATFORMS
|
23
|
+
x86_64-linux
|
24
|
+
|
25
|
+
DEPENDENCIES
|
26
|
+
redis
|
27
|
+
sidekiq
|
28
|
+
workflow_manager!
|
29
|
+
|
30
|
+
BUNDLED WITH
|
31
|
+
2.3.26
|
@@ -1,9 +1,13 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
# example: https://raw.githubusercontent.com/redis/redis/7.0/redis.conf
|
2
|
+
# default, for production
|
3
|
+
port 6379
|
4
|
+
# for test
|
5
|
+
#port 6380
|
6
|
+
loglevel notice
|
7
|
+
logfile ../logs/redis.log
|
3
8
|
databases 5
|
4
|
-
save 300 10
|
9
|
+
save 3600 1 300 10 60 1000
|
5
10
|
rdbcompression yes
|
6
|
-
dir ./dbs
|
7
11
|
dbfilename redis.rdb
|
12
|
+
dir dbs
|
8
13
|
maxmemory 10gb
|
9
|
-
port 6379
|
data/lib/job_checker.rb
CHANGED
@@ -43,15 +43,15 @@ class JobChecker
|
|
43
43
|
end
|
44
44
|
new_job_script
|
45
45
|
end
|
46
|
-
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host)
|
46
|
+
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
|
47
47
|
unless @start_time
|
48
48
|
@start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
49
49
|
end
|
50
50
|
time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
51
|
-
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host].join(',')
|
51
|
+
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
|
52
52
|
end
|
53
53
|
|
54
|
-
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil)
|
54
|
+
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
|
55
55
|
puts "JobID (in JobChecker): #{job_id}"
|
56
56
|
db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
|
57
57
|
db1 = Redis.new(port: PORT, db: 1) # log DB
|
@@ -66,10 +66,10 @@ class JobChecker
|
|
66
66
|
#print ret
|
67
67
|
state = ret.split(/\n/).last.strip
|
68
68
|
#puts "state: #{state}"
|
69
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
69
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
70
70
|
|
71
71
|
unless state == pre_state
|
72
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
72
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
73
73
|
project_jobs = eval((db2[project_id]||[]).to_s)
|
74
74
|
project_jobs = Hash[*project_jobs]
|
75
75
|
project_jobs[job_id] = state
|
@@ -79,10 +79,22 @@ class JobChecker
|
|
79
79
|
pre_state = state
|
80
80
|
sleep WORKER_INTERVAL
|
81
81
|
end while state =~ /RUNNING/ or state =~ /PENDING/ or state =~ /---/
|
82
|
+
|
83
|
+
# post process
|
82
84
|
if next_dataset_id and rails_host
|
83
85
|
uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
|
84
86
|
#p uri
|
85
87
|
res = Net::HTTP.get_response(uri)
|
88
|
+
|
89
|
+
if log_dir and !log_dir.empty?
|
90
|
+
copy_command = copy_command_template.gsub("org_file", log_file)
|
91
|
+
#puts "copy_command=#{copy_command}"
|
92
|
+
system copy_command
|
93
|
+
err_file = log_file.gsub('_o.log','_e.log')
|
94
|
+
copy_command = copy_command_template.gsub("org_file", err_file)
|
95
|
+
#puts "copy_command=#{copy_command}"
|
96
|
+
system copy_command
|
97
|
+
end
|
86
98
|
end
|
87
99
|
end
|
88
100
|
end
|
@@ -187,12 +187,13 @@ module WorkflowManager
|
|
187
187
|
statuses.each do |job_id, status|
|
188
188
|
# puts [job_id, status].join(",")
|
189
189
|
# 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
|
190
|
-
stat, script_basename, time, user, project_number, next_dataset_id, rails_host = status.split(",")
|
190
|
+
stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
|
191
191
|
if stat == "RUNNING" or stat == "PENDING"
|
192
192
|
log_file = logs[job_id]
|
193
193
|
log_puts("JobID (in recovery check): #{job_id}")
|
194
194
|
puts "JobID (in recovery check): #{job_id}"
|
195
|
-
|
195
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
196
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
196
197
|
end
|
197
198
|
end
|
198
199
|
end
|
@@ -304,8 +305,10 @@ module WorkflowManager
|
|
304
305
|
#p log_file
|
305
306
|
#p job_id
|
306
307
|
puts "JobID (in WorkflowManager): #{job_id}"
|
308
|
+
#puts "log_dir=#{log_dir}"
|
307
309
|
sleep 1
|
308
|
-
|
310
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
311
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
309
312
|
job_id
|
310
313
|
end
|
311
314
|
def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
|
data/start_workflow_manager.sh
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
#!/usr/bin/bash
|
2
2
|
source /usr/local/ngseq/etc/lmod_profile
|
3
|
-
module load Dev/Ruby/
|
4
|
-
module load Tools/Redis/
|
3
|
+
module load Dev/Ruby/3.1.3
|
4
|
+
module load Tools/Redis/7.0.8
|
5
5
|
conda activate gtools_env
|
6
6
|
which python
|
7
7
|
which g-sub
|
8
8
|
which g-req
|
9
9
|
mkdir -p logs
|
10
10
|
mkdir -p dbs
|
11
|
-
bundle exec workflow_manager -d druby://fgcz-h-
|
11
|
+
bundle exec workflow_manager -d druby://fgcz-h-031:40001
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workflow_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-02-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -59,6 +59,7 @@ files:
|
|
59
59
|
- ".gitignore"
|
60
60
|
- ".rspec"
|
61
61
|
- Gemfile
|
62
|
+
- Gemfile.lock
|
62
63
|
- LICENSE.txt
|
63
64
|
- README.md
|
64
65
|
- Rakefile
|
@@ -97,7 +98,7 @@ homepage: ''
|
|
97
98
|
licenses:
|
98
99
|
- MIT
|
99
100
|
metadata: {}
|
100
|
-
post_install_message:
|
101
|
+
post_install_message:
|
101
102
|
rdoc_options: []
|
102
103
|
require_paths:
|
103
104
|
- lib
|
@@ -112,8 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
112
113
|
- !ruby/object:Gem::Version
|
113
114
|
version: '0'
|
114
115
|
requirements: []
|
115
|
-
rubygems_version: 3.
|
116
|
-
signing_key:
|
116
|
+
rubygems_version: 3.3.26
|
117
|
+
signing_key:
|
117
118
|
specification_version: 4
|
118
119
|
summary: Workflow Manager manages job submissions using dRuby.
|
119
120
|
test_files:
|