workflow_manager 0.7.5 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a4bcfadba28876bebd8d498893f8b47aac25ac94ec804c7c0a36df737d53db0
-  data.tar.gz: a41195ba36ffc6d28899254d69f89ee15f631c88d4d0d35a94b604f5d922d4cd
+  metadata.gz: bb29ff98b1a798d31a2a6d3bc84b0a2145d975e8b5e30ef242f309eb74edd099
+  data.tar.gz: 8214d4df1d712c8926e771999105201f141c2495a86008c0dd661b45f6826ad3
 SHA512:
-  metadata.gz: c90748c7b70486e627a207ef85ffea4f3f1ad965c2083b15225a86e8731ec6248a2e35512757859f9696d4eb6f2d5dd8aea873f574734f3686e2692fb6e6e31b
-  data.tar.gz: c425daa92e3d7bb94a4d6a3eb30dcbff2e38e5cd0d0d4c4e556beda69d094f94ae9ac6fb34523400ea07ec7c29070ed729dac1e808c50ffd1b71cf46844263ea
+  metadata.gz: 2e9fb82f9743d5545235fab63e95c892d6acb879fd1d13c21ffba15df0cff53bc131a5b89453234e48fc13d468c24dced45f948dfe54f1c28fcb4324b5f758bc
+  data.tar.gz: 6754552645ac73a6609f8e5175f248892a138c0a20392a54fd79de1395c0e7c4e273c3efea1fc984867fcdd3894aa23724026d5699dbcace2bfee441eedc0f09

data/bin/wfm_monitoring

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 # 20121112 masa workflow manager client
-Version = '20210625-165025'
+Version = '20211104-160323'
 
 require 'drb/drb' 
 require 'workflow_manager/optparse_ex'
@@ -13,10 +13,11 @@ opt = OptionParser.new do |o|
   o.on(:server, 'druby://localhost:12345', '-d server', '--server', 'workflow manager URI (default: druby://localhost:12345)')
   o.on(:log, '-o logdir', '--logdir', 'directory of standard output and standard error file outputted after the job')
   o.on(:cores, '-c cores', '--cores', 'Number of cores to request for g-sub')
-  o.on(:nodes, '-n nodes', '--nodes', 'Comma separated list of nodes to submit to for g-sub')
+#  o.on(:nodes, '-n nodes', '--nodes', 'Comma separated list of nodes to submit to for g-sub')
   o.on(:ram, '-r RAM', '--RAM', 'Amount of RAM to request in Gigs for g-sub')
   o.on(:scratch, '-s scratch', '--scratch', 'Amount of scratch space to request in Gigs for g-sub')
-  o.on(:queue, '-q queue', '--queue', 'Queue name')
+#  o.on(:queue, '-q queue', '--queue', 'Queue name')
+  o.on(:nice, '-i nice', '--nice', 'Nice')
   o.parse!(ARGV)
 end
 unless script_file = ARGV[0] and script_file =~ /\.sh/
@@ -49,7 +50,8 @@ sge_options = []
 sge_options << "-c #{opt.cores}" if opt.cores
 sge_options << "-r #{opt.ram}" if opt.ram
 sge_options << "-s #{opt.scratch}" if opt.scratch
-sge_options << "-n #{opt.nodes}" if opt.nodes
+#sge_options << "-n #{opt.nodes}" if opt.nodes
+sge_options << "-i #{opt.nice}" if opt.nice
 
 script_content = File.read(script_file)
 workflow_manager = DRbObject.new_with_uri(uri)

data/config/environments/redis.conf

@@ -1,9 +1,9 @@
 loglevel debug
 logfile ./logs/redis.log
-databases 4
+databases 5
 save 300 10
 rdbcompression yes
 dir ./dbs
 dbfilename redis.rdb
 maxmemory 10gb
-port 6380
+port 6379

data/lib/job_checker.rb

@@ -1,6 +1,9 @@
 require 'sidekiq'
 require 'redis'
 
+require 'uri'
+require 'net/http'
+
 WORKER_INTERVAL = 10 # [s]
 REDIS_CONF = File.expand_path("../../config/environments/redis.conf", __FILE__)
 PORT = if File.exist?(REDIS_CONF)
@@ -40,15 +43,15 @@ class JobChecker
     end
     new_job_script
   end
-  def update_time_status(status, script_basename, user, project_number, next_dataset_id)
+  def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
     unless @start_time
       @start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
     end
     time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
-    [status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id].join(',')
+    [status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
   end
 
-  def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil)
+  def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
     puts "JobID (in JobChecker): #{job_id}"
     db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
     db1 = Redis.new(port: PORT, db: 1) # log DB
@@ -63,10 +66,10 @@ class JobChecker
       #print ret
       state = ret.split(/\n/).last.strip
       #puts "state: #{state}"
-      db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
+      db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
 
       unless state == pre_state
-        db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
+        db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
         project_jobs = eval((db2[project_id]||[]).to_s)
         project_jobs = Hash[*project_jobs]
         project_jobs[job_id] = state
@@ -76,6 +79,23 @@ class JobChecker
       pre_state = state
       sleep WORKER_INTERVAL
     end while state =~ /RUNNING/  or state =~ /PENDING/ or state =~ /---/
+
+    # post process
+    if next_dataset_id and rails_host
+      uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
+      #p uri
+      res = Net::HTTP.get_response(uri)
+
+      if log_dir and !log_dir.empty?
+        copy_command = copy_command_template.gsub("org_file", log_file)
+        #puts "copy_command=#{copy_command}"
+        system copy_command
+        err_file = log_file.gsub('_o.log','_e.log')
+        copy_command = copy_command_template.gsub("org_file", err_file)
+        #puts "copy_command=#{copy_command}"
+        system copy_command
+      end
+    end
   end
 end
 

data/lib/workflow_manager/cluster.rb

@@ -450,11 +450,15 @@ module WorkflowManager
       partition = if i = options.index("-p")
                     options[i+1]
                   end
+      nice = if i = options.index("-i")
+               options[i+1]
+             end
       new_options = []
       new_options << "--mem=#{ram}G" if ram
       new_options << "-n #{cores}" if cores
       new_options << "--tmp=#{scratch}G" if scratch
       new_options << "-p #{partition}" if partition
+      new_options << "--nice=#{nice}" if nice
       new_options.join(" ")
     end
     def submit_job(script_file, script_content, option='')

data/lib/workflow_manager/server.rb

@@ -163,6 +163,7 @@ module WorkflowManager
                     RedisDB.new(1, @redis_conf)
                 end
       @jobs = RedisDB.new(2, @redis_conf)
+      @trees = RedisDB.new(4, @redis_conf)
 
       @system_log = File.join(@log_dir, "system.log")
       @mutex = Mutex.new
@@ -186,12 +187,13 @@ module WorkflowManager
         statuses.each do |job_id, status|
           # puts [job_id, status].join(",")
           # 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
-          stat, script_basename, time, user, project_number, next_dataset_id = status.split(",")
+          stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
           if stat == "RUNNING" or stat == "PENDING"
             log_file = logs[job_id]
             log_puts("JobID (in recovery check): #{job_id}")
             puts "JobID (in recovery check): #{job_id}"
-            JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id)
+            copy_command_template = copy_commands("org_file", log_dir).first
+            job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
           end
         end
       end
@@ -296,15 +298,17 @@ module WorkflowManager
         Thread.current.kill
       end
     end
-    def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='', next_dataset_id='')
+    def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='', next_dataset_id='', rails_host=nil)
       script_basename = File.basename(script_path)
       job_id, log_file, command = @cluster.submit_job(script_path, script_content, sge_options)
       #p command
       #p log_file
       #p job_id
       puts "JobID (in WorkflowManager): #{job_id}"
+      #puts "log_dir=#{log_dir}"
       sleep 1
-      JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id)
+      copy_command_template = copy_commands("org_file", log_dir).first
+      job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
       job_id
     end
     def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
@@ -555,6 +559,19 @@ module WorkflowManager
     def cluster_node_list
       @cluster.node_list
     end
+    def save_dataset_tree(project_number, json)
+      @trees.transaction do |trees|
+        trees[project_number] = json
+      end
+      json
+    end
+    def load_dataset_tree(project_number)
+      json = nil
+      @trees.transaction do |trees|
+        json = trees[project_number]
+      end
+      json
+    end
   end
 end
 

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.7.5"
+  VERSION = "0.7.9"
 end

data/start_workflow_manager.sh

@@ -8,4 +8,4 @@ which g-sub
 which g-req
 mkdir -p logs
 mkdir -p dbs
-bundle exec workflow_manager -d druby://fgcz-h-032:40001
+bundle exec workflow_manager -d druby://fgcz-h-032:40002

data/test/job_list.rb

@@ -1,13 +1,13 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
-# Version = '20210723-134812'
+# Version = '20211001-104513'
 
 PORT = (ARGV[0]||6380).to_i
 require 'redis'
 db0 = Redis.new(port: PORT, db: 0)
 db1 = Redis.new(port: PORT, db: 1)
 db2 = Redis.new(port: PORT, db: 2)
-#db3 = Redis.new(port: 6380, db: 3)
+db4 = Redis.new(port: PORT, db: 4)
 
 class Redis
   def show_all
@@ -18,8 +18,8 @@ class Redis
   end
 end
 
-dbs = [db0, db1, db2]
-db_notes = ["state DB", "log DB", "project job DB"]
+dbs = [db0, db1, db2, db4]
+db_notes = ["state DB", "log DB", "project job DB", "JS tree DB"]
 
 dbs.each.with_index do |db, i|
   note = db_notes[i]
@@ -48,3 +48,10 @@ db2.keys.sort.each do |key|
   value = db2.get(key)
   puts [key, value].join("\t")
 end
+
+puts
+puts "db3, status DB3, project specific"
+db3.keys.sort.each do |key|
+  value = db3.get(key)
+  puts [key, value].join("\t")
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.9
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler