workflow_manager 0.7.8 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8c60e97cd0b4f5c15220a65c77e296b90c38bcdf0218adea5700d6e71556c494
4
- data.tar.gz: 201c8eb50cf7647bf4fa53abbd64586ec05b118a82e0fb053f4102977b3c6cb8
3
+ metadata.gz: 5668a0ae734ac5ac90ffc891559681da2412107a29cbc8a5b279c9ac9201a27e
4
+ data.tar.gz: aa34a9a4ef006005e98d375520adfd2a8126ff835f76ff0a2d4432f439149248
5
5
  SHA512:
6
- metadata.gz: c976d0a6b684536d78657b3d21b9cd682e51cbefdbd11c00ed4f65be802f3be749d2e25a0cdf61c620a7153f112131626511af9fc434f0fe24886f0b688cbd8a
7
- data.tar.gz: ca7beaebea2c29ce6da4dcb15185258aac2f22b57f773017f8f876b36c81643b212d7e09edd528051ee0695138b945fbd13e5201da4e54c4eadada605d15a07b
6
+ metadata.gz: 42e691408c30e9f29321199423a25fbb3844f109cf2169074262cd3a2d08bd5b9f62e1bf759e3f0448fa25fdd592606be77b7a3789cf9c1f3e3ee411122d194e
7
+ data.tar.gz: 0572fa546e578c279d6efdb8c63dac03c2075fa5bedea5c854d49a42e1c0df7e54525ab0af9ae2c54bf567f5d39b13883b1f8ab2553595872829a286546ff8ca
data/.gitignore CHANGED
@@ -3,7 +3,6 @@
3
3
  .bundle
4
4
  .config
5
5
  .yardoc
6
- Gemfile.lock
7
6
  InstalledFiles
8
7
  _yardoc
9
8
  coverage
data/Gemfile.lock ADDED
@@ -0,0 +1,31 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ workflow_manager (0.7.9)
5
+
6
+ GEM
7
+ remote: https://rubygems.org/
8
+ specs:
9
+ concurrent-ruby (1.2.0)
10
+ connection_pool (2.3.0)
11
+ rack (3.0.4.1)
12
+ redis (5.0.6)
13
+ redis-client (>= 0.9.0)
14
+ redis-client (0.12.1)
15
+ connection_pool
16
+ sidekiq (7.0.5)
17
+ concurrent-ruby (< 2)
18
+ connection_pool (>= 2.3.0)
19
+ rack (>= 2.2.4)
20
+ redis-client (>= 0.11.0)
21
+
22
+ PLATFORMS
23
+ x86_64-linux
24
+
25
+ DEPENDENCIES
26
+ redis
27
+ sidekiq
28
+ workflow_manager!
29
+
30
+ BUNDLED WITH
31
+ 2.3.26
@@ -1,9 +1,13 @@
1
- loglevel debug
2
- logfile ./logs/redis.log
1
+ # example: https://raw.githubusercontent.com/redis/redis/7.0/redis.conf
2
+ # default, for production
3
+ port 6379
4
+ # for test
5
+ #port 6380
6
+ loglevel notice
7
+ logfile ../logs/redis.log
3
8
  databases 5
4
- save 300 10
9
+ save 3600 1 300 10 60 1000
5
10
  rdbcompression yes
6
- dir ./dbs
7
11
  dbfilename redis.rdb
12
+ dir dbs
8
13
  maxmemory 10gb
9
- port 6379
data/lib/job_checker.rb CHANGED
@@ -43,15 +43,15 @@ class JobChecker
43
43
  end
44
44
  new_job_script
45
45
  end
46
- def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host)
46
+ def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
47
47
  unless @start_time
48
48
  @start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
49
49
  end
50
50
  time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
51
- [status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host].join(',')
51
+ [status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
52
52
  end
53
53
 
54
- def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil)
54
+ def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
55
55
  puts "JobID (in JobChecker): #{job_id}"
56
56
  db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
57
57
  db1 = Redis.new(port: PORT, db: 1) # log DB
@@ -66,10 +66,10 @@ class JobChecker
66
66
  #print ret
67
67
  state = ret.split(/\n/).last.strip
68
68
  #puts "state: #{state}"
69
- db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
69
+ db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
70
70
 
71
71
  unless state == pre_state
72
- db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
72
+ db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
73
73
  project_jobs = eval((db2[project_id]||[]).to_s)
74
74
  project_jobs = Hash[*project_jobs]
75
75
  project_jobs[job_id] = state
@@ -79,10 +79,22 @@ class JobChecker
79
79
  pre_state = state
80
80
  sleep WORKER_INTERVAL
81
81
  end while state =~ /RUNNING/ or state =~ /PENDING/ or state =~ /---/
82
+
83
+ # post process
82
84
  if next_dataset_id and rails_host
83
85
  uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
84
86
  #p uri
85
87
  res = Net::HTTP.get_response(uri)
88
+
89
+ if log_dir and !log_dir.empty?
90
+ copy_command = copy_command_template.gsub("org_file", log_file)
91
+ #puts "copy_command=#{copy_command}"
92
+ system copy_command
93
+ err_file = log_file.gsub('_o.log','_e.log')
94
+ copy_command = copy_command_template.gsub("org_file", err_file)
95
+ #puts "copy_command=#{copy_command}"
96
+ system copy_command
97
+ end
86
98
  end
87
99
  end
88
100
  end
@@ -187,12 +187,13 @@ module WorkflowManager
187
187
  statuses.each do |job_id, status|
188
188
  # puts [job_id, status].join(",")
189
189
  # 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
190
- stat, script_basename, time, user, project_number, next_dataset_id, rails_host = status.split(",")
190
+ stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
191
191
  if stat == "RUNNING" or stat == "PENDING"
192
192
  log_file = logs[job_id]
193
193
  log_puts("JobID (in recovery check): #{job_id}")
194
194
  puts "JobID (in recovery check): #{job_id}"
195
- JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host)
195
+ copy_command_template = copy_commands("org_file", log_dir).first
196
+ job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
196
197
  end
197
198
  end
198
199
  end
@@ -304,8 +305,10 @@ module WorkflowManager
304
305
  #p log_file
305
306
  #p job_id
306
307
  puts "JobID (in WorkflowManager): #{job_id}"
308
+ #puts "log_dir=#{log_dir}"
307
309
  sleep 1
308
- JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host)
310
+ copy_command_template = copy_commands("org_file", log_dir).first
311
+ job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
309
312
  job_id
310
313
  end
311
314
  def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
@@ -1,3 +1,3 @@
1
1
  module WorkflowManager
2
- VERSION = "0.7.8"
2
+ VERSION = "0.8.0"
3
3
  end
@@ -1,11 +1,11 @@
1
1
  #!/usr/bin/bash
2
2
  source /usr/local/ngseq/etc/lmod_profile
3
- module load Dev/Ruby/2.6.7
4
- module load Tools/Redis/6.0.1
3
+ module load Dev/Ruby/3.1.3
4
+ module load Tools/Redis/7.0.8
5
5
  conda activate gtools_env
6
6
  which python
7
7
  which g-sub
8
8
  which g-req
9
9
  mkdir -p logs
10
10
  mkdir -p dbs
11
- bundle exec workflow_manager -d druby://fgcz-h-032:40002
11
+ bundle exec workflow_manager -d druby://fgcz-h-031:40001
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: workflow_manager
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.8
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Functional Genomics Center Zurich
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-11-04 00:00:00.000000000 Z
11
+ date: 2023-02-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -59,6 +59,7 @@ files:
59
59
  - ".gitignore"
60
60
  - ".rspec"
61
61
  - Gemfile
62
+ - Gemfile.lock
62
63
  - LICENSE.txt
63
64
  - README.md
64
65
  - Rakefile
@@ -97,7 +98,7 @@ homepage: ''
97
98
  licenses:
98
99
  - MIT
99
100
  metadata: {}
100
- post_install_message:
101
+ post_install_message:
101
102
  rdoc_options: []
102
103
  require_paths:
103
104
  - lib
@@ -112,8 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
112
113
  - !ruby/object:Gem::Version
113
114
  version: '0'
114
115
  requirements: []
115
- rubygems_version: 3.0.3.1
116
- signing_key:
116
+ rubygems_version: 3.3.26
117
+ signing_key:
117
118
  specification_version: 4
118
119
  summary: Workflow Manager manages job submissions using dRuby.
119
120
  test_files: