workflow_manager 0.7.8 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c60e97cd0b4f5c15220a65c77e296b90c38bcdf0218adea5700d6e71556c494
-  data.tar.gz: 201c8eb50cf7647bf4fa53abbd64586ec05b118a82e0fb053f4102977b3c6cb8
+  metadata.gz: 5668a0ae734ac5ac90ffc891559681da2412107a29cbc8a5b279c9ac9201a27e
+  data.tar.gz: aa34a9a4ef006005e98d375520adfd2a8126ff835f76ff0a2d4432f439149248
 SHA512:
-  metadata.gz: c976d0a6b684536d78657b3d21b9cd682e51cbefdbd11c00ed4f65be802f3be749d2e25a0cdf61c620a7153f112131626511af9fc434f0fe24886f0b688cbd8a
-  data.tar.gz: ca7beaebea2c29ce6da4dcb15185258aac2f22b57f773017f8f876b36c81643b212d7e09edd528051ee0695138b945fbd13e5201da4e54c4eadada605d15a07b
+  metadata.gz: 42e691408c30e9f29321199423a25fbb3844f109cf2169074262cd3a2d08bd5b9f62e1bf759e3f0448fa25fdd592606be77b7a3789cf9c1f3e3ee411122d194e
+  data.tar.gz: 0572fa546e578c279d6efdb8c63dac03c2075fa5bedea5c854d49a42e1c0df7e54525ab0af9ae2c54bf567f5d39b13883b1f8ab2553595872829a286546ff8ca

data/.gitignore

@@ -3,7 +3,6 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    workflow_manager (0.7.9)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.2.0)
+    connection_pool (2.3.0)
+    rack (3.0.4.1)
+    redis (5.0.6)
+      redis-client (>= 0.9.0)
+    redis-client (0.12.1)
+      connection_pool
+    sidekiq (7.0.5)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      rack (>= 2.2.4)
+      redis-client (>= 0.11.0)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  redis
+  sidekiq
+  workflow_manager!
+
+BUNDLED WITH
+   2.3.26

data/config/environments/redis.conf

@@ -1,9 +1,13 @@
-loglevel debug
-logfile ./logs/redis.log
+# example: https://raw.githubusercontent.com/redis/redis/7.0/redis.conf
+# default, for production
+port 6379
+# for test
+#port 6380
+loglevel notice
+logfile ../logs/redis.log
 databases 5
-save 300 10
+save 3600 1 300 10 60 1000
 rdbcompression yes
-dir ./dbs
 dbfilename redis.rdb
+dir dbs
 maxmemory 10gb
-port 6379

data/lib/job_checker.rb

@@ -43,15 +43,15 @@ class JobChecker
     end
     new_job_script
   end
-  def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host)
+  def update_time_status(status, script_basename, user, project_number, next_dataset_id, rails_host, log_dir)
     unless @start_time
       @start_time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
     end
     time = Time.now.strftime("%Y-%m-%d %H:%M:%S")
-    [status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host].join(',')
+    [status, script_basename, [@start_time, time].join("/"), user, project_number, next_dataset_id, rails_host, log_dir].join(',')
   end
 
-  def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil)
+  def perform(job_id, script_basename, log_file, user, project_id, next_dataset_id=nil, rails_host=nil, log_dir=nil, copy_command_template=nil)
     puts "JobID (in JobChecker): #{job_id}"
     db0 = Redis.new(port: PORT, db: 0) # state + alpha DB
     db1 = Redis.new(port: PORT, db: 1) # log DB
@@ -66,10 +66,10 @@ class JobChecker
       #print ret
       state = ret.split(/\n/).last.strip
       #puts "state: #{state}"
-      db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
+      db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
 
       unless state == pre_state
-        db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host)
+        db0[job_id] = update_time_status(state, script_basename, user, project_id, next_dataset_id, rails_host, log_dir)
         project_jobs = eval((db2[project_id]||[]).to_s)
         project_jobs = Hash[*project_jobs]
         project_jobs[job_id] = state
@@ -79,10 +79,22 @@ class JobChecker
       pre_state = state
       sleep WORKER_INTERVAL
     end while state =~ /RUNNING/  or state =~ /PENDING/ or state =~ /---/
+
+    # post process
     if next_dataset_id and rails_host
       uri = URI("#{rails_host}/data_set/#{next_dataset_id}/update_completed_samples")
       #p uri
       res = Net::HTTP.get_response(uri)
+
+      if log_dir and !log_dir.empty?
+        copy_command = copy_command_template.gsub("org_file", log_file)
+        #puts "copy_command=#{copy_command}"
+        system copy_command
+        err_file = log_file.gsub('_o.log','_e.log')
+        copy_command = copy_command_template.gsub("org_file", err_file)
+        #puts "copy_command=#{copy_command}"
+        system copy_command
+      end
     end
   end
 end

data/lib/workflow_manager/server.rb

@@ -187,12 +187,13 @@ module WorkflowManager
         statuses.each do |job_id, status|
           # puts [job_id, status].join(",")
           # 120249,RUNNING,QC_ventricles_100k.sh,2021-07-30 09:47:04/2021-07-30 09:47:04,masaomi,1535
-          stat, script_basename, time, user, project_number, next_dataset_id, rails_host = status.split(",")
+          stat, script_basename, time, user, project_number, next_dataset_id, rails_host, log_dir = status.split(",")
           if stat == "RUNNING" or stat == "PENDING"
             log_file = logs[job_id]
             log_puts("JobID (in recovery check): #{job_id}")
             puts "JobID (in recovery check): #{job_id}"
-            JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host)
+            copy_command_template = copy_commands("org_file", log_dir).first
+            job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
           end
         end
       end
@@ -304,8 +305,10 @@ module WorkflowManager
       #p log_file
       #p job_id
       puts "JobID (in WorkflowManager): #{job_id}"
+      #puts "log_dir=#{log_dir}"
       sleep 1
-      JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host)
+      copy_command_template = copy_commands("org_file", log_dir).first
+      job_checker = JobChecker.perform_async(job_id, script_basename, log_file, user, project_number, next_dataset_id, rails_host, log_dir, copy_command_template)
       job_id
     end
     def start_monitoring2(script_path, script_content, user='sushi_lover', project_number=0, sge_options='', log_dir='')

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.7.8"
+  VERSION = "0.8.0"
 end

data/start_workflow_manager.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/bash
 source /usr/local/ngseq/etc/lmod_profile
-module load Dev/Ruby/2.6.7
-module load Tools/Redis/6.0.1
+module load Dev/Ruby/3.1.3
+module load Tools/Redis/7.0.8
 conda activate gtools_env
 which python
 which g-sub
 which g-req
 mkdir -p logs
 mkdir -p dbs
-bundle exec workflow_manager -d druby://fgcz-h-032:40002
+bundle exec workflow_manager -d druby://fgcz-h-031:40001

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.8.0
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2023-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -59,6 +59,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -97,7 +98,7 @@ homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -112,8 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: Workflow Manager manages job submissions using dRuby.
 test_files: