workflow_manager 0.7.8 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/Gemfile.lock +31 -0
- data/config/environments/redis.conf +9 -5
- data/lib/job_checker.rb +17 -5
- data/lib/workflow_manager/server.rb +6 -3
- data/lib/workflow_manager/version.rb +1 -1
- data/start_workflow_manager.sh +3 -3
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5668a0ae734ac5ac90ffc891559681da2412107a29cbc8a5b279c9ac9201a27e
|
4
|
+
data.tar.gz: aa34a9a4ef006005e98d375520adfd2a8126ff835f76ff0a2d4432f439149248
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42e691408c30e9f29321199423a25fbb3844f109cf2169074262cd3a2d08bd5b9f62e1bf759e3f0448fa25fdd592606be77b7a3789cf9c1f3e3ee411122d194e
|
7
|
+
data.tar.gz: 0572fa546e578c279d6efdb8c63dac03c2075fa5bedea5c854d49a42e1c0df7e54525ab0af9ae2c54bf567f5d39b13883b1f8ab2553595872829a286546ff8ca
|
data/.gitignore
CHANGED
data/Gemfile.lock
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
workflow_manager (0.7.9)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
concurrent-ruby (1.2.0)
|
10
|
+
connection_pool (2.3.0)
|
11
|
+
rack (3.0.4.1)
|
12
|
+
redis (5.0.6)
|
13
|
+
redis-client (>= 0.9.0)
|
14
|
+
redis-client (0.12.1)
|
15
|
+
connection_pool
|
16
|
+
sidekiq (7.0.5)
|
17
|
+
concurrent-ruby (< 2)
|
18
|
+
connection_pool (>= 2.3.0)
|
19
|
+
rack (>= 2.2.4)
|
20
|
+
redis-client (>= 0.11.0)
|
21
|
+
|
22
|
+
PLATFORMS
|
23
|
+
x86_64-linux
|
24
|
+
|
25
|
+
DEPENDENCIES
|
26
|
+
redis
|
27
|
+
sidekiq
|
28
|
+
workflow_manager!
|
29
|
+
|
30
|
+
BUNDLED WITH
|
31
|
+
2.3.26
|
@@ -1,9 +1,13 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
# example: https://raw.githubusercontent.com/redis/redis/7.0/redis.conf
|
2
|
+
# default, for production
|
3
|
+
port 6379
|
4
|
+
# for test
|
5
|
+
#port 6380
|
6
|
+
loglevel notice
|
7
|
+
logfile ../logs/redis.log
|
3
8
|
databases 5
|
4
|
-
save 300 10
|
9
|
+
save 3600 1 300 10 60 1000
|
5
10
|
rdbcompression yes
|
6
|
-
dir ./dbs
|
7
11
|
dbfilename redis.rdb
|
12
|
+
dir dbs
|
8
13
|
maxmemory 10gb
|
9
|
-
port 6379
|
data/lib/job_checker.rb
CHANGED
@@ -43,15 +43,15 @@ class JobChecker
|
|
43
43
|
end
|
44
44
|
new_job_script
|
45
45
|
end
|
46
|
-
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host)
|
46
|
+
def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
|
47
47
|
unless @start_time
|
48
48
|
@start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
49
49
|
end
|
50
50
|
time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
51
|
-
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host].join(',')
|
51
|
+
[status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
|
52
52
|
end
|
53
53
|
|
54
|
-
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil)
|
54
|
+
def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
|
55
55
|
puts "JobID (in JobChecker): #{job_id}"
|
56
56
|
db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
|
57
57
|
db1 = Redis.new(port: PORT, db: 1) # log DB
|
@@ -66,10 +66,10 @@ class JobChecker
|
|
66
66
|
#print ret
|
67
67
|
state = ret.split(/\n/).last.strip
|
68
68
|
#puts "state: #{state}"
|
69
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
69
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
70
70
|
|
71
71
|
unless state == pre_state
|
72
|
-
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
|
72
|
+
db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
|
73
73
|
project_jobs = eval((db2[project_id]||[]).to_s)
|
74
74
|
project_jobs = Hash[*project_jobs]
|
75
75
|
project_jobs[job_id] = state
|
@@ -79,10 +79,22 @@ class JobChecker
|
|
79
79
|
pre_state = state
|
80
80
|
sleep WORKER_INTERVAL
|
81
81
|
end while state =~ /RUNNING/ or state =~ /PENDING/ or state =~ /---/
|
82
|
+
|
83
|
+
# post process
|
82
84
|
if next_dataset_id and rails_host
|
83
85
|
uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
|
84
86
|
#p uri
|
85
87
|
res = Net::HTTP.get_response(uri)
|
88
|
+
|
89
|
+
if log_dir and !log_dir.empty?
|
90
|
+
copy_command = copy_command_template.gsub("org_file", log_file)
|
91
|
+
#puts "copy_command=#{copy_command}"
|
92
|
+
system copy_command
|
93
|
+
err_file = log_file.gsub('_o.log','_e.log')
|
94
|
+
copy_command = copy_command_template.gsub("org_file", err_file)
|
95
|
+
#puts "copy_command=#{copy_command}"
|
96
|
+
system copy_command
|
97
|
+
end
|
86
98
|
end
|
87
99
|
end
|
88
100
|
end
|
@@ -187,12 +187,13 @@ module WorkflowManager
|
|
187
187
|
statuses.each do |job_id, status|
|
188
188
|
# puts [job_id, status].join(",")
|
189
189
|
# 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
|
190
|
-
stat, script_basename, time, user, project_number, next_dataset_id, rails_host = status.split(",")
|
190
|
+
stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
|
191
191
|
if stat == "RUNNING" or stat == "PENDING"
|
192
192
|
log_file = logs[job_id]
|
193
193
|
log_puts("JobID (in recovery check): #{job_id}")
|
194
194
|
puts "JobID (in recovery check): #{job_id}"
|
195
|
-
|
195
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
196
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
196
197
|
end
|
197
198
|
end
|
198
199
|
end
|
@@ -304,8 +305,10 @@ module WorkflowManager
|
|
304
305
|
#p log_file
|
305
306
|
#p job_id
|
306
307
|
puts "JobID (in WorkflowManager): #{job_id}"
|
308
|
+
#puts "log_dir=#{log_dir}"
|
307
309
|
sleep 1
|
308
|
-
|
310
|
+
copy_command_template = copy_commands("org_file", log_dir).first
|
311
|
+
job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
|
309
312
|
job_id
|
310
313
|
end
|
311
314
|
def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
|
data/start_workflow_manager.sh
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
#!/usr/bin/bash
|
2
2
|
source /usr/local/ngseq/etc/lmod_profile
|
3
|
-
module load Dev/Ruby/
|
4
|
-
module load Tools/Redis/
|
3
|
+
module load Dev/Ruby/3.1.3
|
4
|
+
module load Tools/Redis/7.0.8
|
5
5
|
conda activate gtools_env
|
6
6
|
which python
|
7
7
|
which g-sub
|
8
8
|
which g-req
|
9
9
|
mkdir -p logs
|
10
10
|
mkdir -p dbs
|
11
|
-
bundle exec workflow_manager -d druby://fgcz-h-
|
11
|
+
bundle exec workflow_manager -d druby://fgcz-h-031:40001
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workflow_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-02-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -59,6 +59,7 @@ files:
|
|
59
59
|
- ".gitignore"
|
60
60
|
- ".rspec"
|
61
61
|
- Gemfile
|
62
|
+
- Gemfile.lock
|
62
63
|
- LICENSE.txt
|
63
64
|
- README.md
|
64
65
|
- Rakefile
|
@@ -97,7 +98,7 @@ homepage: ''
|
|
97
98
|
licenses:
|
98
99
|
- MIT
|
99
100
|
metadata: {}
|
100
|
-
post_install_message:
|
101
|
+
post_install_message:
|
101
102
|
rdoc_options: []
|
102
103
|
require_paths:
|
103
104
|
- lib
|
@@ -112,8 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
112
113
|
- !ruby/object:Gem::Version
|
113
114
|
version: '0'
|
114
115
|
requirements: []
|
115
|
-
rubygems_version: 3.
|
116
|
-
signing_key:
|
116
|
+
rubygems_version: 3.3.26
|
117
|
+
signing_key:
|
117
118
|
specification_version: 4
|
118
119
|
summary: Workflow Manager manages job submissions using dRuby.
|
119
120
|
test_files:
|