workflow_manager 0.7.1 → 0.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ea3816ec54cac7bf1097d94f410aeb9d17641755494de742814c4151a20d4de
-  data.tar.gz: 8adcaf1765299151ccddc6f68740989e265b68b7829c07a4e49ee751bb0ff608
+  metadata.gz: 5a4bcfadba28876bebd8d498893f8b47aac25ac94ec804c7c0a36df737d53db0
+  data.tar.gz: a41195ba36ffc6d28899254d69f89ee15f631c88d4d0d35a94b604f5d922d4cd
 SHA512:
-  metadata.gz: fb11c23c1346ca6fff218b2283b6254cf18ef3872725c1f7a068c8261d1af2217d7d13686c61ea28f2d4c01b7c1f1b242862f9ae60e223ffb64f70d0646aae2b
-  data.tar.gz: 764e474e5fbbf9fffb411f5484c774bff60e517893ef983e7307ca923d8c12f982ac4c4f20468015d7b78a2b0ff4c9adab8cbf465c0a9fb09bf46170bd9eea69
+  metadata.gz: c90748c7b70486e627a207ef85ffea4f3f1ad965c2083b15225a86e8731ec6248a2e35512757859f9696d4eb6f2d5dd8aea873f574734f3686e2692fb6e6e31b
+  data.tar.gz: c425daa92e3d7bb94a4d6a3eb30dcbff2e38e5cd0d0d4c4e556beda69d094f94ae9ac6fb34523400ea07ec7c29070ed729dac1e808c50ffd1b71cf46844263ea

data/Gemfile

@@ -2,4 +2,5 @@ source 'https://rubygems.org'
 
 gem 'redis'
 gem 'sidekiq'
-gem 'workflow_manager', :path => '/srv/GT/analysis/masaomi/FGCZ/prototype_workflow_manager_with_sidekiq_20210122/workflow_manager/'
+gem 'workflow_manager', :path => './'
+#gem 'workflow_manager'

data/bin/workflow_manager

@@ -33,7 +33,7 @@ if opt.mode
     default_config_file = File.join(default_config_dir, opt.mode+".rb")
     if File.exist?(default_config_file)
       default_redis_config_file = File.join(default_config_dir, "redis.conf")
-      default_sidekiq_config_file = File.join(default_config_dir, "sidekiq.conf")
+      default_sidekiq_config_file = File.join(default_config_dir, "sidekiq.yml")
       FileUtils.cp(default_redis_config_file, config_dir)
       FileUtils.cp(default_sidekiq_config_file, config_dir)
       FileUtils.cp(default_config_file, config_file)

data/lib/job_checker.rb

@@ -40,15 +40,15 @@ class JobChecker
     end
     new_job_script
   end
-  def update_time_status(status, script_basename, user, project_number)
+  def update_time_status(status, script_basename, user, project_number, next_dataset_id)
     unless @start_time
       @start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
     end
     time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
-    [status, script_basename, [@start_time, time].join("/"), user, project_number].join(',')
+    [status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id].join(',')
   end
 
-  def perform(job_id, script_basename, log_file, user, project_id)
+  def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil)
     puts "JobID (in JobChecker): #{job_id}"
     db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
     db1 = Redis.new(port: PORT, db: 1) # log DB
@@ -63,10 +63,10 @@ class JobChecker
       #print ret
       state = ret.split(/\n/).last.strip
       #puts "state: #{state}"
-      db0[job_id] = update_time_status(state, script_basename, user, project_id)
+      db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
 
       unless state == pre_state
-        db0[job_id] = update_time_status(state, script_basename, user, project_id)
+        db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id)
         project_jobs = eval((db2[project_id]||[]).to_s)
         project_jobs = Hash[*project_jobs]
         project_jobs[job_id] = state

data/lib/workflow_manager/server.rb

@@ -177,6 +177,7 @@ module WorkflowManager
       log_puts("Cluster = #{@cluster.name}")
       log_puts("Server starts")
       log_puts("Recovery check")
+      sleep 2
       recovery_job_checker
     end
     def recovery_job_checker
@@ -185,12 +186,12 @@ module WorkflowManager
         statuses.each do |job_id, status|
           # puts [job_id, status].join(",")
           # 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
-          stat, script_basename, time, user, project_number = status.split(",")
+          stat, script_basename, time, user, project_number, next_dataset_id = status.split(",")
           if stat == "RUNNING" or stat == "PENDING"
             log_file = logs[job_id]
             log_puts("JobID (in recovery check): #{job_id}")
             puts "JobID (in recovery check): #{job_id}"
-            JobChecker.perform_async(job_id, script_basename, log_file, user, project_number)
+            JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id)
           end
         end
       end
@@ -295,7 +296,7 @@ module WorkflowManager
         Thread.current.kill
       end
     end
-    def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
+    def start_monitoring3(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='', next_dataset_id='')
       script_basename = File.basename(script_path)
       job_id, log_file, command = @cluster.submit_job(script_path, script_content, sge_options)
       #p command
@@ -303,7 +304,7 @@ module WorkflowManager
       #p job_id
       puts "JobID (in WorkflowManager): #{job_id}"
       sleep 1
-      JobChecker.perform_async(job_id, script_basename, log_file, user, project_number)
+      JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id)
       job_id
     end
     def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')
@@ -472,17 +473,23 @@ module WorkflowManager
       job_idsh = if job_ids
                    Hash[*(job_ids.split(',')).map{|job_id| [job_id, true]}.flatten]
                  end
-      s_ = {}
-      unless job_ids
+      if project_number
+        s_ = {}
         @jobs.transaction do |jobs|
           if project_jobs = jobs[project_number]
             s_ = Hash[*eval(project_jobs)]
           end
         end
-      end
-      @statuses.transaction do |statuses|
-        s_.each do |job_id, stat|
-          s << [job_id, statuses[job_id]]
+        @statuses.transaction do |statuses|
+          s_.each do |job_id, stat|
+            s << [job_id, statuses[job_id]]
+          end
+        end
+      else
+        @statuses.transaction do |statuses|
+          statuses.each do |key, value|
+            s << [key, value]
+          end
         end
       end
       if job_ids

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.7.1"
+  VERSION = "0.7.5"
 end

data/start_workflow_manager.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/bash
+source /usr/local/ngseq/etc/lmod_profile
+module load Dev/Ruby/2.6.7
+module load Tools/Redis/6.0.1
+conda activate gtools_env
+which python
+which g-sub
+which g-req
+mkdir -p logs
+mkdir -p dbs
+bundle exec workflow_manager -d druby://fgcz-h-032:40001

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.5
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-30 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -87,6 +87,7 @@ files:
 - spec/cluster_spec.rb
 - spec/server_spec.rb
 - spec/spec_helper.rb
+- start_workflow_manager.sh
 - test/call_worker4.rb
 - test/call_worker_method.rb
 - test/job_list.rb