workflow_manager 0.5.1 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 11c00265553f6ee24f60116455ad308ce4f880ca
-  data.tar.gz: 7a76ec915498d09ec76e8504ffd9c53bf2b68954
+SHA256:
+  metadata.gz: 624dc0f37286ee8b893ceb0d73cce4ce83ab6078be6b1f06a96c316e145311f4
+  data.tar.gz: 805777d8e45242281c3e929b03a92d3fd9205c6c091f716ebb16b93bf15904e9
 SHA512:
-  metadata.gz: fd80735df1170477400d0501bd4c5407598f48560c6af06d3acadc82f93c95f64371b0b0829681710712476b3465063eda608bf89122f83457664676de819f9c
-  data.tar.gz: f4809de592cddf922cb6c7d3b0d85e3936503d1bf97c82e96107f1a7dcf5984f09b7abd72abe50f56c1a1db9234bcf7e454f44ec78875c2ad95d2ea5b3bbe7ed
+  metadata.gz: 8c89da95c15863910c397bc149b799b094ee631feb092a7e0ffea7648c2148e09677b1288bf41daa5aa798b8a315ad3e7d2a4686d2b7bb5ced4f57fe8eff2161
+  data.tar.gz: bea9060c0b8322b3b096ffdae4ac82d9f523bc9322b8bfa412f986aab5c56881523985698c974eb581560929eea60d34189724efae29c28e31127512f0b56478

data/bin/wfm_hello

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 # 20121112 masa workflow manager client
-Version = '20130517-111334'
+Version = '20200522-134606'
 
 require 'drb/drb' 
 
@@ -28,3 +28,4 @@ if wfmrc
 end
 workflow_manager = DRbObject.new_with_uri(uri)
 puts workflow_manager.hello
+puts workflow_manager.cluster_node_list

data/lib/workflow_manager/cluster.rb

@@ -38,6 +38,8 @@ module WorkflowManager
     end
     def default_node
     end
+    def node_list
+    end
   end
 
   class LocalComputer < Cluster
@@ -238,6 +240,63 @@ module WorkflowManager
         'fgcz-h-010: cpu 8,mem  30 GB,scr 400G' => 'fgcz-h-010',
       }
     end
+    def node_list
+      node2scr = {}
+      command = "qhost -F scratch"
+      keep = nil
+      IO.popen(command) do |out|
+        while line=out.gets
+          hostname, arch, ncpu, loading, memtot, memuse, *others = line.split
+          if hostname =~ /fgcz/
+            keep = hostname
+          elsif scratch_ = line.chomp.split.last and
+                scratch = scratch_.split('=').last
+            node2scr[keep] = scratch.to_i
+            keep = nil
+          end
+        end
+      end
+
+      list = {}
+      keep = nil
+      command = 'qhost -q'
+      IO.popen(command) do |out|
+        while line=out.gets
+          # HOSTNAME                ARCH         NCPU  LOAD  MEMTOT  MEMUSE  SWAPTO  SWAPUS
+          hostname, arch, ncpu, loading, memtot, memuse, *others = line.split
+          if hostname =~ /fgcz/
+            #puts [hostname, ncpu, loading, memtot, memuse].join("\t")
+            mem = memtot.gsub(/G/, '').to_i
+            keep = [hostname, ncpu, "#{mem}G"]
+          elsif hostname == "GT" and keep and cores = line.chomp.split.last and cores !~ /[du]/
+            hostname = keep.shift
+            keep[0] = cores
+            if scr = node2scr[hostname] and scr >= 1000
+              scr = "%.1f" % (scr.to_f / 1000)
+              scr << "T"
+            else
+              scr = scr.to_s + "G"
+            end
+            keep << scr
+            list[hostname] = keep
+            keep = nil
+          end
+        end
+      end
+
+      # reformat
+      nodes = {}
+      list.each do |hostname, specs|
+        # 20190823 masa tentatively off use f47
+        unless hostname =~ /fgcz-c-047/
+          cores, ram, scr = specs
+          key = "#{hostname}: cores #{cores}, ram #{ram}, scr #{scr}"
+          value = hostname
+          nodes[key] = value
+        end
+      end
+      nodes
+    end
   end
 
   class FGCZCourseCluster < FGCZCluster
@@ -290,4 +349,180 @@ module WorkflowManager
       }
     end
   end
+
+  class FGCZDevian10Cluster < Cluster
+    def submit_job(script_file, script_content, option='')
+      if script_name = File.basename(script_file) and script_name =~ /\.sh/
+        script_name = script_name.split(/\.sh/).first + ".sh"
+        new_job_script = generate_new_job_script(script_name, script_content)
+        new_job_script_base = File.basename(new_job_script)
+        log_file = File.join(@log_dir, new_job_script_base + "_o.log")
+        err_file = File.join(@log_dir, new_job_script_base + "_e.log")
+        command = "g-sub -o #{log_file} -e #{err_file} -q course #{option} #{new_job_script}"
+        #command = "sbatch -o #{log_file} -e #{err_file} #{new_job_script}"
+        job_id = `#{command}`
+        #job_id = job_id.match(/Your job (\d+) \(/)[1]
+        job_id = job_id.chomp.split.last
+        [job_id, log_file, command]
+      else
+        err_msg = "FGCZDevian10Cluster#submit_job, ERROR: script_name is not *.sh: #{File.basename(script_file)}"
+        warn err_msg
+        raise err_msg
+      end
+    end
+    def job_running?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          # ["JOBID", "PARTITION", "NAME", "USER", "ST", "TIME", "NODES", "NODELIST(REASON)"]
+          # ["206", "employee", "test.sh", "masaomi", "R", "0:03", "1", "fgcz-h-030"]
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state == 'R'
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def job_ends?(log_file)
+      log_flag = false
+      IO.popen("tail -n 10 #{log_file} 2> /dev/null") do |io|
+        while line=io.gets
+          if line =~ /__SCRIPT END__/
+            log_flag = true
+            break
+          end
+        end
+      end
+      log_flag
+    end
+    def job_pending?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state =~ /PD/
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def copy_commands(org_dir, dest_parent_dir, now=nil)
+      commands = if now == "force"
+                   target_file = File.join(dest_parent_dir, File.basename(org_dir))
+                   ["g-req copynow -f #{org_dir} #{dest_parent_dir}"]
+                 elsif now
+                   ["g-req copynow #{org_dir} #{dest_parent_dir}"]
+                 else
+                   ["g-req -w copy #{org_dir} #{dest_parent_dir}"]
+                 end
+    end
+    def kill_command(job_id)
+      command = "scancel #{job_id}"
+    end
+    def delete_command(target)
+      command = "g-req remove #{target}"
+    end
+    def cluster_nodes
+      nodes = {
+        'fgcz-h-900: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-900',
+        'fgcz-h-901: cpu 8,mem  30 GB,scr 400G' => 'fgcz-h-901',
+      }
+    end
+  end
+
+  class FGCZDebian10CourseCluster < Cluster
+    def submit_job(script_file, script_content, option='')
+      if script_name = File.basename(script_file) and script_name =~ /\.sh/
+        script_name = script_name.split(/\.sh/).first + ".sh"
+        new_job_script = generate_new_job_script(script_name, script_content)
+        new_job_script_base = File.basename(new_job_script)
+        log_file = File.join(@log_dir, new_job_script_base + "_o.log")
+        err_file = File.join(@log_dir, new_job_script_base + "_e.log")
+        command = "g-sub -o #{log_file} -e #{err_file} -q course #{option} #{new_job_script}"
+        job_id = `#{command}`
+        job_id = job_id.chomp.split.last
+        [job_id, log_file, command]
+      else
+        err_msg = "FGCZDebian10CourseCluster#submit_job, ERROR: script_name is not *.sh: #{File.basename(script_file)}"
+        warn err_msg
+        raise err_msg
+      end
+    end
+    def job_running?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          # ["JOBID", "PARTITION", "NAME", "USER", "ST", "TIME", "NODES", "NODELIST(REASON)"]
+          # ["206", "employee", "test.sh", "masaomi", "R", "0:03", "1", "fgcz-h-030"]
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state == 'R'
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def job_ends?(log_file)
+      log_flag = false
+      IO.popen("tail -n 10 #{log_file} 2> /dev/null") do |io|
+        while line=io.gets
+          if line =~ /__SCRIPT END__/
+            log_flag = true
+            break
+          end
+        end
+      end
+      log_flag
+    end
+    def job_pending?(job_id)
+     qstat_flag = false
+      IO.popen('squeue') do |io|
+        while line=io.gets
+          jobid, partition, name, user, state, *others = line.chomp.split
+          if jobid.strip == job_id and state =~ /PD/
+            qstat_flag = true
+            break
+          end
+        end
+      end
+      qstat_flag
+    end
+    def copy_commands(org_dir, dest_parent_dir, now=nil)
+      commands = ["cp -r #{org_dir} #{dest_parent_dir}"]
+    end
+    def kill_command(job_id)
+      command = "scancel #{job_id}"
+    end
+    def delete_command(target)
+      command = "rm -rf #{target}"
+    end
+    def cluster_nodes
+      nodes = {
+        'fgcz-h-900: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-900',
+        'fgcz-h-901: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-901',
+        'fgcz-h-902: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-902',
+        'fgcz-h-903: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-903',
+        'fgcz-h-904: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-904',
+        'fgcz-h-905: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-905',
+        'fgcz-h-906: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-906',
+        'fgcz-h-907: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-907',
+        'fgcz-h-908: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-908',
+        'fgcz-h-909: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-909',
+        'fgcz-h-910: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-910',
+        'fgcz-h-911: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-911',
+        'fgcz-h-912: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-912',
+        'fgcz-h-913: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-913',
+        'fgcz-h-914: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-914',
+        'fgcz-h-915: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-915',
+        'fgcz-h-916: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-916',
+        'fgcz-h-917: cpu 8,mem  30 GB,scr 500G' => 'fgcz-h-917',
+      }
+    end
+  end
+
 end

data/lib/workflow_manager/server.rb

@@ -100,6 +100,7 @@ module WorkflowManager
       end
     end
     class RedisDB
+      attr_accessor :port
       def run_redis_server(redis_conf)
         @pid = fork do
           exec("redis-server #{redis_conf}")
@@ -108,11 +109,13 @@ module WorkflowManager
           Process.waitpid @pid
         end
       end
-      def initialize(db_no=0, redis_conf=nil)
+      def initialize(db_no=0, redis_conf)
         if db_no==0
           run_redis_server(redis_conf)
         end
-        @db = Redis.new(db: db_no)
+        conf = Hash[*CSV.readlines(redis_conf, col_sep: " ").map{|a| [a.first, a[1,100].join(",")]}.flatten]
+        @port = (conf["port"]||6379).to_i
+        @db = Redis.new(port: @port, db: db_no)
       end
       def transaction
         #@db.multi do
@@ -154,7 +157,7 @@ module WorkflowManager
                   when "KyotoCabinet"
                     KyotoDB.new(@db_logs)
                   when "Redis"
-                    RedisDB.new(1)
+                    RedisDB.new(1, @redis_conf)
                 end
 
       @system_log = File.join(@log_dir, "system.log")
@@ -163,6 +166,7 @@ module WorkflowManager
       puts("DB = #{DB_MODE}")
       if DB_MODE == "Redis"
         puts("Redis conf = #{config.redis_conf}")
+        puts("Redis port = #{@logs.port}")
       end
       puts("Cluster = #{@cluster.name}")
       log_puts("DB = #{DB_MODE}")
@@ -505,6 +509,9 @@ module WorkflowManager
             end
     end
     alias_method :check_status, :success_or_fail
+    def cluster_node_list
+      @cluster.node_list
+    end
   end
 end
 

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.5.1"
+  VERSION = "0.5.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.6
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-26 00:00:00.000000000 Z
+date: 2020-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -104,8 +104,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Workflow Manager manages job submissions using dRuby.