workflow_manager 0.5.3 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5711738c7423585b2db31fd502f0f1dd44ac23ead6f77ae2a8bfa328d4c2649
-  data.tar.gz: e2a981b0065fa8e6efacb8ad4433162809bd4533f32e72e554cf98459b7f85c1
+  metadata.gz: 2a2b2e23d038d7a8e71a88f3dfab605a10ce889a4bc59ec4fcd7ec647c7239c8
+  data.tar.gz: 44e6121985b273019fc5098efd6084736935149496ca6aa38e710f9f4ae5e48f
 SHA512:
-  metadata.gz: 6fece378a564236133c30296a6b1a240dc830fb21441960c58ad1ccf9874da9eac96e78f17181cb95f1f51f1645bf124c4253b8f6abd3f871318470f2de22b6e
-  data.tar.gz: 15333f88e1e38458d1c963731519e35d8971f4047ea07fd7c1821fea1f291b2c4a862a693fbca0e1b4a8ab56d7435dc82eff56f4d0c89f00b09dd2749d539e4a
+  metadata.gz: 78ee76362caff7fe5f6eb668cae5858e05dd94b06e29dc8c754ce424c1c3af8c8d21d87ac4c4de8a40b17c7ad10dafd0492b6a465677d191f8e599d5db96e787
+  data.tar.gz: bd14b2c039458901783a4485c6aad6afbcf5f20a1fef89496e9e93db14d072425990b0b2629c2d05b0b1d8920d4917d46f2062c0f37a1d02e3861d891abc3742

data/bin/wfm_hello

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 # 20121112 masa workflow manager client
-Version = '20130517-111334'
+Version = '20200522-134606'
 
 require 'drb/drb' 
 
@@ -28,3 +28,4 @@ if wfmrc
 end
 workflow_manager = DRbObject.new_with_uri(uri)
 puts workflow_manager.hello
+puts workflow_manager.cluster_node_list

data/bin/wfm_monitoring

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 # 20121112 masa workflow manager client
-Version = '20160317-153614'
+Version = '20200722-161135'
 
 require 'drb/drb' 
 require 'workflow_manager/optparse_ex'
@@ -16,6 +16,7 @@ opt = OptionParser.new do |o|
   o.on(:nodes, '-n nodes', '--nodes', 'Comma separated list of nodes to submit to for g-sub')
   o.on(:ram, '-r RAM', '--RAM', 'Amount of RAM to request in Gigs for g-sub')
   o.on(:scratch, '-s scratch', '--scratch', 'Amount of scratch space to request in Gigs for g-sub')
+  o.on(:queue, '-q queue', '--queue', 'Queue name')
   o.parse!(ARGV)
 end
 unless script_file = ARGV[0] and script_file =~ /\.sh/

data/lib/workflow_manager/cluster.rb

@@ -278,7 +278,7 @@ module WorkflowManager
               scr = scr.to_s + "G"
             end
             keep << scr
-            list[hostname] = keep 
+            list[hostname] = keep
             keep = nil
           end
         end
@@ -349,4 +349,285 @@ module WorkflowManager
       }
     end
   end
+
+  class FGCZDevian10Cluster < Cluster
+    def submit_job(script_file, script_content, option='')
+      if script_name = File.basename(script_file) and script_name =~ /\.sh/
+        script_name = script_name.split(/\.sh/).first + ".sh"
+        new_job_script = generate_new_job_script(script_name, script_content)
+        new_job_script_base = File.basename(new_job_script)
+        log_file = File.join(@log_dir, new_job_script_base + "_o.log")
+        err_file = File.join(@log_dir, new_job_script_base + "_e.log")
+        command = "g-sub -o #{log_file} -e #{err_file} -q course #{option} #{new_job_script}"
+        #command = "sbatch -o #{log_file} -e #{err_file} #{new_job_script}"
+        job_id = `#{command}`
+        #job_id = job_id.match(/Your job (\d+) \(/)[1]
+        job_id = job_id.chomp.split.last
+        [job_id, log_file, command]
+      else
+        err_msg = "FGCZDevian10Cluster#submit_job, ERROR: script_name is not *.sh: #{File.basename(script_file)}"
+        warn err_msg
+        raise err_msg
+      end
+    end
+    def job_running?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          # ["JOBID", "PARTITION", "NAME", "USER", "ST", "TIME", "NODES", "NODELIST(REASON)"]
+          # ["206", "employee", "test.sh", "masaomi", "R", "0:03", "1", "fgcz-h-030"]
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state == 'R'
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def job_ends?(log_file)
+      log_flag = false
+      IO.popen("tail -n 10 #{log_file} 2> /dev/null") do |io|
+        while line=io.gets
+          if line =~ /__SCRIPT END__/
+            log_flag = true
+            break
+          end
+        end
+      end
+      log_flag
+    end
+    def job_pending?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state =~ /PD/
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def copy_commands(org_dir, dest_parent_dir, now=nil)
+      commands = if now == "force"
+                   target_file = File.join(dest_parent_dir, File.basename(org_dir))
+                   ["g-req copynow -f #{org_dir} #{dest_parent_dir}"]
+                 elsif now
+                   ["g-req copynow #{org_dir} #{dest_parent_dir}"]
+                 else
+                   ["g-req -w copy #{org_dir} #{dest_parent_dir}"]
+                 end
+    end
+    def kill_command(job_id)
+      command = "scancel #{job_id}"
+    end
+    def delete_command(target)
+      command = "g-req remove #{target}"
+    end
+    def cluster_nodes
+      nodes = {
+        'fgcz-h-900: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-900',
+        'fgcz-h-901: cpu 8,mem  30 GB,scr 400G' => 'fgcz-h-901',
+      }
+    end
+  end
+
+  class FGCZDebian10CourseCluster < Cluster
+    def submit_job(script_file, script_content, option='')
+      if script_name = File.basename(script_file) and script_name =~ /\.sh/
+        script_name = script_name.split(/\.sh/).first + ".sh"
+        new_job_script = generate_new_job_script(script_name, script_content)
+        new_job_script_base = File.basename(new_job_script)
+        log_file = File.join(@log_dir, new_job_script_base + "_o.log")
+        err_file = File.join(@log_dir, new_job_script_base + "_e.log")
+        command = "g-sub -o #{log_file} -e #{err_file} -q course #{option} #{new_job_script}"
+        job_id = `#{command}`
+        job_id = job_id.chomp.split.last
+        [job_id, log_file, command]
+      else
+        err_msg = "FGCZDebian10CourseCluster#submit_job, ERROR: script_name is not *.sh: #{File.basename(script_file)}"
+        warn err_msg
+        raise err_msg
+      end
+    end
+    def job_running?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          # ["JOBID", "PARTITION", "NAME", "USER", "ST", "TIME", "NODES", "NODELIST(REASON)"]
+          # ["206", "employee", "test.sh", "masaomi", "R", "0:03", "1", "fgcz-h-030"]
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state == 'R'
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def job_ends?(log_file)
+      log_flag = false
+      IO.popen("tail -n 10 #{log_file} 2> /dev/null") do |io|
+        while line=io.gets
+          if line =~ /__SCRIPT END__/
+            log_flag = true
+            break
+          end
+        end
+      end
+      log_flag
+    end
+    def job_pending?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state =~ /PD/
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def copy_commands(org_dir, dest_parent_dir, now=nil)
+      commands = ["cp -r #{org_dir} #{dest_parent_dir}"]
+    end
+    def kill_command(job_id)
+      command = "scancel #{job_id}"
+    end
+    def delete_command(target)
+      command = "rm -rf #{target}"
+    end
+    def cluster_nodes
+      nodes = {
+        'fgcz-h-900: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-900',
+        'fgcz-h-901: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-901',
+        'fgcz-h-902: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-902',
+        'fgcz-h-903: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-903',
+        'fgcz-h-904: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-904',
+        'fgcz-h-905: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-905',
+        'fgcz-h-906: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-906',
+        'fgcz-h-907: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-907',
+        'fgcz-h-908: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-908',
+        'fgcz-h-909: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-909',
+        'fgcz-h-910: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-910',
+        'fgcz-h-911: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-911',
+        'fgcz-h-912: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-912',
+        'fgcz-h-913: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-913',
+        'fgcz-h-914: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-914',
+        'fgcz-h-915: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-915',
+        'fgcz-h-916: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-916',
+        'fgcz-h-917: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-917',
+      }
+    end
+  end
+
+  class FGCZDebian10Cluster < Cluster
+    def parse(options)
+      options = options.split
+      ram = if i = options.index("-r")
+              options[i+1]
+            end
+      cores = if i = options.index("-c")
+                options[i+1]
+              end
+      scratch = if i = options.index("-s")
+                  options[i+1]
+                end
+      queue = if i = options.index("-q")
+                options[i+1]
+              end
+      new_options = []
+      new_options << "--mem=#{ram}G" if ram
+      new_options << "-n #{cores}" if cores
+      new_options << "--tmp=#{scratch}G" if scratch
+      new_options << "-p #{queue}" if queue
+      new_options.join(" ")
+    end
+    def submit_job(script_file, script_content, option='')
+      if script_name = File.basename(script_file) and script_name =~ /\.sh/
+        script_name = script_name.split(/\.sh/).first + ".sh"
+        new_job_script = generate_new_job_script(script_name, script_content)
+        new_job_script_base = File.basename(new_job_script)
+        log_file = File.join(@log_dir, new_job_script_base + "_o.log")
+        err_file = File.join(@log_dir, new_job_script_base + "_e.log")
+        #command = "g-sub -o #{log_file} -e #{err_file} -q user #{option} #{new_job_script}"
+        sbatch_options = parse(option)
+        command = "sbatch -o #{log_file} -e #{err_file} -N 1 #{sbatch_options} #{new_job_script}"
+        puts command
+        job_id = `#{command}`
+        job_id = job_id.chomp.split.last
+        [job_id, log_file, command]
+      else
+        err_msg = "FGCZDebian10Cluster#submit_job, ERROR: script_name is not *.sh: #{File.basename(script_file)}"
+        warn err_msg
+        raise err_msg
+      end
+    end
+    def job_running?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          # ["JOBID", "PARTITION", "NAME", "USER", "ST", "TIME", "NODES", "NODELIST(REASON)"]
+          # ["206", "employee", "test.sh", "masaomi", "R", "0:03", "1", "fgcz-h-030"]
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state == 'R'
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def job_ends?(log_file)
+      log_flag = false
+      IO.popen("tail -n 10 #{log_file} 2> /dev/null") do |io|
+        while line=io.gets
+          if line =~ /__SCRIPT END__/
+            log_flag = true
+            break
+          end
+        end
+      end
+      log_flag
+    end
+    def job_pending?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state =~ /PD/
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def copy_commands(org_dir, dest_parent_dir, now=nil)
+      commands = if now == "force"
+                   target_file = File.join(dest_parent_dir, File.basename(org_dir))
+                   ["g-req copynow -f #{org_dir} #{dest_parent_dir}"]
+                 elsif now
+                   ["g-req copynow #{org_dir} #{dest_parent_dir}"]
+                 else
+                   ["g-req -w copy #{org_dir} #{dest_parent_dir}"]
+                 end
+    end
+    def kill_command(job_id)
+      command = "scancel #{job_id}"
+    end
+    def delete_command(target)
+      command = "g-req remove #{target}"
+    end
+    def cluster_nodes
+      nodes = {
+        'fgcz-h-110: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-110',
+        'fgcz-h-111: cpu 8,mem  30 GB,scr 400G' => 'fgcz-h-111',
+      }
+    end
+  end
 end

data/lib/workflow_manager/server.rb

@@ -509,9 +509,9 @@ module WorkflowManager
             end
     end
     alias_method :check_status, :success_or_fail
-  end
-  def cluster_node_list
-    @cluster.node_list
+    def cluster_node_list
+      @cluster.node_list
+    end
   end
 end
 

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.5.3"
+  VERSION = "0.5.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.8
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-22 00:00:00.000000000 Z
+date: 2020-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler