workflow_manager 0.5.6 → 0.5.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/wfm_monitoring +2 -1
- data/lib/workflow_manager/cluster.rb +105 -0
- data/lib/workflow_manager/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ede215b5a867ebbafc9af7ead5a9afa04ed1530eac7d0d1bffb2ee42b09cbded
|
4
|
+
data.tar.gz: 7f68d53462f3c22ec889edaec171a6d5685063090b28264dfd93a224ca2d6f95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dcc2bac302faf22e7170143d7377bb87144b3e6ccf049ba8d768318fd2f3a7ccdb213643c55fadbddfcdedd92b103e572dd75310d94c27fa917f55fd1c9d2c87
|
7
|
+
data.tar.gz: 18789821274de2c8961f15e6a49194c1d4e2c2f09eab095e9f7e14f575388507ff83a0dff5da480bcb79ec276956eea85739ee4d3f8cfd12c6c44139cdc3b465
|
data/bin/wfm_monitoring
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
# encoding: utf-8
|
3
3
|
# 20121112 masa workflow manager client
|
4
|
-
Version = '
|
4
|
+
Version = '20200722-161135'
|
5
5
|
|
6
6
|
require 'drb/drb'
|
7
7
|
require 'workflow_manager/optparse_ex'
|
@@ -16,6 +16,7 @@ opt = OptionParser.new do |o|
|
|
16
16
|
o.on(:nodes, '-n nodes', '--nodes', 'Comma separated list of nodes to submit to for g-sub')
|
17
17
|
o.on(:ram, '-r RAM', '--RAM', 'Amount of RAM to request in Gigs for g-sub')
|
18
18
|
o.on(:scratch, '-s scratch', '--scratch', 'Amount of scratch space to request in Gigs for g-sub')
|
19
|
+
o.on(:queue, '-q queue', '--queue', 'Queue name')
|
19
20
|
o.parse!(ARGV)
|
20
21
|
end
|
21
22
|
unless script_file = ARGV[0] and script_file =~ /\.sh/
|
@@ -525,4 +525,109 @@ module WorkflowManager
|
|
525
525
|
end
|
526
526
|
end
|
527
527
|
|
528
|
+
class FGCZDebian10Cluster < Cluster
|
529
|
+
def parse(options)
|
530
|
+
options = options.split
|
531
|
+
ram = if i = options.index("-r")
|
532
|
+
options[i+1]
|
533
|
+
end
|
534
|
+
cores = if i = options.index("-c")
|
535
|
+
options[i+1]
|
536
|
+
end
|
537
|
+
scratch = if i = options.index("-s")
|
538
|
+
options[i+1]
|
539
|
+
end
|
540
|
+
queue = if i = options.index("-q")
|
541
|
+
options[i+1]
|
542
|
+
end
|
543
|
+
new_options = []
|
544
|
+
new_options << "--mem=#{ram}G" if ram
|
545
|
+
new_options << "-n #{cores}" if cores
|
546
|
+
new_options << "--tmp=#{scratch}G" if scratch
|
547
|
+
new_options << "-p #{queue}" if queue
|
548
|
+
new_options.join(" ")
|
549
|
+
end
|
550
|
+
def submit_job(script_file, script_content, option='')
|
551
|
+
if script_name = File.basename(script_file) and script_name =~ /\.sh/
|
552
|
+
script_name = script_name.split(/\.sh/).first + ".sh"
|
553
|
+
new_job_script = generate_new_job_script(script_name, script_content)
|
554
|
+
new_job_script_base = File.basename(new_job_script)
|
555
|
+
log_file = File.join(@log_dir, new_job_script_base + "_o.log")
|
556
|
+
err_file = File.join(@log_dir, new_job_script_base + "_e.log")
|
557
|
+
#command = "g-sub -o #{log_file} -e #{err_file} -q user #{option} #{new_job_script}"
|
558
|
+
sbatch_options = parse(option)
|
559
|
+
command = "sbatch -o #{log_file} -e #{err_file} #{sbatch_options} #{new_job_script}"
|
560
|
+
puts command
|
561
|
+
job_id = `#{command}`
|
562
|
+
job_id = job_id.chomp.split.last
|
563
|
+
[job_id, log_file, command]
|
564
|
+
else
|
565
|
+
err_msg = "FGCZDebian10Cluster#submit_job, ERROR: script_name is not *.sh: #{File.basename(script_file)}"
|
566
|
+
warn err_msg
|
567
|
+
raise err_msg
|
568
|
+
end
|
569
|
+
end
|
570
|
+
def job_running?(job_id)
|
571
|
+
qstat_flag = false
|
572
|
+
IO.popen('squeue') do |io|
|
573
|
+
while line=io.gets
|
574
|
+
# ["JOBID", "PARTITION", "NAME", "USER", "ST", "TIME", "NODES", "NODELIST(REASON)"]
|
575
|
+
# ["206", "employee", "test.sh", "masaomi", "R", "0:03", "1", "fgcz-h-030"]
|
576
|
+
jobid, partition, name, user, state, *others = line.chomp.split
|
577
|
+
if jobid.strip == job_id and state == 'R'
|
578
|
+
qstat_flag = true
|
579
|
+
break
|
580
|
+
end
|
581
|
+
end
|
582
|
+
end
|
583
|
+
qstat_flag
|
584
|
+
end
|
585
|
+
def job_ends?(log_file)
|
586
|
+
log_flag = false
|
587
|
+
IO.popen("tail -n 10 #{log_file} 2> /dev/null") do |io|
|
588
|
+
while line=io.gets
|
589
|
+
if line =~ /__SCRIPT END__/
|
590
|
+
log_flag = true
|
591
|
+
break
|
592
|
+
end
|
593
|
+
end
|
594
|
+
end
|
595
|
+
log_flag
|
596
|
+
end
|
597
|
+
def job_pending?(job_id)
|
598
|
+
qstat_flag = false
|
599
|
+
IO.popen('squeue') do |io|
|
600
|
+
while line=io.gets
|
601
|
+
jobid, partition, name, user, state, *others = line.chomp.split
|
602
|
+
if jobid.strip == job_id and state =~ /PD/
|
603
|
+
qstat_flag = true
|
604
|
+
break
|
605
|
+
end
|
606
|
+
end
|
607
|
+
end
|
608
|
+
qstat_flag
|
609
|
+
end
|
610
|
+
def copy_commands(org_dir, dest_parent_dir, now=nil)
|
611
|
+
commands = if now == "force"
|
612
|
+
target_file = File.join(dest_parent_dir, File.basename(org_dir))
|
613
|
+
["g-req copynow -f #{org_dir} #{dest_parent_dir}"]
|
614
|
+
elsif now
|
615
|
+
["g-req copynow #{org_dir} #{dest_parent_dir}"]
|
616
|
+
else
|
617
|
+
["g-req -w copy #{org_dir} #{dest_parent_dir}"]
|
618
|
+
end
|
619
|
+
end
|
620
|
+
def kill_command(job_id)
|
621
|
+
command = "scancel #{job_id}"
|
622
|
+
end
|
623
|
+
def delete_command(target)
|
624
|
+
command = "g-req remove #{target}"
|
625
|
+
end
|
626
|
+
def cluster_nodes
|
627
|
+
nodes = {
|
628
|
+
'fgcz-h-110: cpu 8,mem 30 GB,scr 500G' => 'fgcz-h-110',
|
629
|
+
'fgcz-h-111: cpu 8,mem 30 GB,scr 400G' => 'fgcz-h-111',
|
630
|
+
}
|
631
|
+
end
|
632
|
+
end
|
528
633
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: workflow_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Functional Genomics Center Zurich
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|