workflow_manager 0.7.9 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb29ff98b1a798d31a2a6d3bc84b0a2145d975e8b5e30ef242f309eb74edd099
-  data.tar.gz: 8214d4df1d712c8926e771999105201f141c2495a86008c0dd661b45f6826ad3
+  metadata.gz: 46119a1d4da028cc76b16a56545512607afa923742d982f2b5f1941ce42bacd3
+  data.tar.gz: 922c122276a8e6842017288a643eb4dbe208b4f69907f690d4a8c00adb1649b3
 SHA512:
-  metadata.gz: 2e9fb82f9743d5545235fab63e95c892d6acb879fd1d13c21ffba15df0cff53bc131a5b89453234e48fc13d468c24dced45f948dfe54f1c28fcb4324b5f758bc
-  data.tar.gz: 6754552645ac73a6609f8e5175f248892a138c0a20392a54fd79de1395c0e7c4e273c3efea1fc984867fcdd3894aa23724026d5699dbcace2bfee441eedc0f09
+  metadata.gz: 1a2e0eea78ab77f45b1b80e558f880ef362c2668cb3d30b67039dc078fd45a27d49401d802c46caeb55352053b25f22a05f7d50d5ff0b6d970e22e8fdd73b043
+  data.tar.gz: ec74fee4bebb6c4014535e228a1c2c6b229a8a7151c540c9fb9853635a207e05a177ec9f8dcce4cf7f5eb331294a6ef1eed213f6893404155d030c4595ada2c9

data/.gitignore

@@ -3,7 +3,6 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    workflow_manager (0.8.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
+    rack (3.0.8)
+    redis (5.0.6)
+      redis-client (>= 0.9.0)
+    redis-client (0.17.0)
+      connection_pool
+    sidekiq (7.1.4)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      rack (>= 2.2.4)
+      redis-client (>= 0.14.0)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  redis
+  sidekiq
+  workflow_manager!
+
+BUNDLED WITH
+   2.3.26

data/config/environments/redis.conf

@@ -1,9 +1,13 @@
-loglevel debug
-logfile ./logs/redis.log
+# example: https://raw.githubusercontent.com/redis/redis/7.0/redis.conf
+# default, for production
+port 6379
+# for test
+#port 6380
+loglevel notice
+logfile ../logs/redis.log
 databases 5
-save 300 10
+save 3600 1 300 10 60 1000
 rdbcompression yes
-dir ./dbs
 dbfilename redis.rdb
+dir dbs
 maxmemory 10gb
-port 6379

data/lib/workflow_manager/cluster.rb

@@ -28,7 +28,7 @@ module WorkflowManager
     end
     def job_pending?(job_id)
     end
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
     end
     def kill_command(job_id)
     end
@@ -75,7 +75,7 @@ module WorkflowManager
       result = `#{command}`
       result.to_s.empty? ? false : true
     end
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
       commands = []
       commands << "mkdir -p #{dest_parent_dir}"
       commands << "cp -r #{org_dir} #{dest_parent_dir}"
@@ -199,7 +199,7 @@ module WorkflowManager
       end
       qstat_flag
     end
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
       commands = if now == "force"
                    target_file = File.join(dest_parent_dir, File.basename(org_dir))
                    ["g-req copynow -f #{org_dir} #{dest_parent_dir}"]
@@ -300,7 +300,7 @@ module WorkflowManager
   end
 
   class FGCZCourseCluster < FGCZCluster
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
       commands = ["cp -r #{org_dir} #{dest_parent_dir}"]
     end
     def delete_command(target)
@@ -333,7 +333,7 @@ module WorkflowManager
     def job_pending?(job_id)
       # TODO
     end
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
       # TODO
     end
     def kill_command(job_id)
@@ -411,7 +411,7 @@ module WorkflowManager
       end
       qstat_flag
     end
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
       commands = if now == "force"
                    target_file = File.join(dest_parent_dir, File.basename(org_dir))
                    ["g-req copynow -f #{org_dir} #{dest_parent_dir}"]
@@ -521,14 +521,16 @@ module WorkflowManager
       end
       qstat_flag
     end
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
       commands = if now == "force"
                    target_file = File.join(dest_parent_dir, File.basename(org_dir))
                    ["g-req copynow -f #{org_dir} #{dest_parent_dir}"]
                  elsif now
                    ["g-req copynow #{org_dir} #{dest_parent_dir}"]
-                 else
+                 elsif queue.nil? or queue == "light"
                    ["g-req -w copy #{org_dir} #{dest_parent_dir}"]
+                 else
+                   ["g-req -w copy -f heavy #{org_dir} #{dest_parent_dir}"]
                  end
     end
     def kill_command(job_id)
@@ -546,7 +548,7 @@ module WorkflowManager
   end
 
   class FGCZDebian10DemoCluster < FGCZDebian10Cluster
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
       commands = ["cp -r #{org_dir} #{dest_parent_dir}"]
     end
     def delete_command(target)

data/lib/workflow_manager/server.rb

@@ -202,8 +202,8 @@ module WorkflowManager
     def hello
       'hello hoge hoge bar boo bundle, '+ @cluster.name
     end
-    def copy_commands(org_dir, dest_parent_dir, now=nil)
-      @cluster.copy_commands(org_dir, dest_parent_dir, now)
+    def copy_commands(org_dir, dest_parent_dir, now=nil, queue="light")
+      @cluster.copy_commands(org_dir, dest_parent_dir, now, queue)
     end
     def kill_job(job_id)
       status(job_id, 'FAIL')

data/lib/workflow_manager/version.rb

@@ -1,3 +1,3 @@
 module WorkflowManager
-  VERSION = "0.7.9"
+  VERSION = "0.8.1"
 end

data/start_workflow_manager.sh

@@ -1,11 +1,12 @@
 #!/usr/bin/bash
 source /usr/local/ngseq/etc/lmod_profile
-module load Dev/Ruby/2.6.7
-module load Tools/Redis/6.0.1
-conda activate gtools_env
+module load Dev/Ruby/3.1.3
+module load Tools/Redis/7.0.8
+#conda activate gtools_env
 which python
 which g-sub
 which g-req
 mkdir -p logs
 mkdir -p dbs
-bundle exec workflow_manager -d druby://fgcz-h-032:40002
+#bundle exec workflow_manager -d druby://fgcz-h-031:40001
+bundle exec workflow_manager -d druby://fgcz-h-032:40001

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow_manager
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.8.1
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-03 00:00:00.000000000 Z
+date: 2023-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -59,6 +59,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -97,7 +98,7 @@ homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -112,8 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: Workflow Manager manages job submissions using dRuby.
 test_files: