sushi_fabric 0.9.6 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 326d811109da5893c3ac6ea35f946f18849e1bb4
-  data.tar.gz: adce9f2299a8b16217cfd715f82ad0f894f3b0d9
+SHA256:
+  metadata.gz: 6bb6ed61655c7decac6439f7bd3c73784888ae9686d2611c10f9d4c40bda0621
+  data.tar.gz: 91e98688ac53cfc55560decb2537ee1b38e9c93d24ffb4cfae02875088e2e910
 SHA512:
-  metadata.gz: 9369616e25ddabc40f335f5490bb925591cb52f642e6326cccfbe2205bc807be81983e1f530993ae0256488f3903fc3b3c47e09a2a6f0d261e93e475f712921e
-  data.tar.gz: fedae1c8626fb161831a5e2989c9aeab0cccf54459d09996339f1005322908a4a368326804fec7cee52639fd43272f71d3703de4180640ad3aebe52514ea04a8
+  metadata.gz: d2ffd594a92358285b0996f0fa06053af38cb65cfe337049b0e619f7e590fdd8ae91b005adfa50c088ced30b7c8235dd7f6d469f36284af62e5a94f5bab7d4b7
+  data.tar.gz: b5ee9600a1e1b2fe1d76b27bc37cc30a230a5cafb0e613fb5ab13f587ddade33e8f365c87be81e315239d78fb90c99bffe9f65d5ede95f62c8947c1c540be75a

data/lib/sushi_fabric/sushiApp.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
-# Version = '20200409-110502'
+# Version = '20201105-115625'
 
 require 'csv'
 require 'fileutils'
@@ -237,6 +237,7 @@ class SushiApp
     @params['ram'] = nil
     @params['scratch'] = nil
     @params['node'] = ''
+    @params['queue'] = ''
     @params['process_mode'] = 'SAMPLE'
     @params['samples'] = ''
     @job_ids = []
@@ -389,16 +390,18 @@ class SushiApp
     FileUtils.mkdir_p(@scratch_result_dir)
     FileUtils.mkdir_p(@job_script_dir)
   end
-  def check_latest_module_version(mod)
-    command_out =  %x[ bash -lc "source #{@module_source}; module whatis #{mod} 2>&1" ]
-    latest_mod = nil
-    command_out.split("\n").each do |line|
-      if line =~ /#{mod}/
-        latest_mod = line.split.first
-        break
+  def check_latest_modules_version(modules)
+    command_out =  %x[ bash -lc "source #{@module_source}; module whatis #{modules.join(" ")} 2>&1" | cut -f 1 -d " " | sort | uniq ]
+    latest_modules = []
+    command_out.split("\n").each do |line_|
+      line = line_.chomp
+      unless line.empty?
+        if line =~ /#{modules.join("|")}/
+          latest_modules << line
+        end
       end
     end
-    latest_mod
+    latest_modules
   end
   def job_header
     @scratch_dir = if @params['process_mode'] == 'SAMPLE'
@@ -418,9 +421,17 @@ class SushiApp
                        ""
                      end
     module_add_commands = if @modules and !@modules.empty?
-                            modules_with_version = @modules.map{|mod| check_latest_module_version(mod)}
-                            modules_with_version.compact!
-                            "module add #{modules_with_version.join(' ')}"
+                            modules_with_version = check_latest_modules_version(@modules)
+                            if @modules.length == modules_with_version.length
+                              modules_with_version.compact!
+                              "module add #{modules_with_version.join(' ')}"
+                            else
+                              @logger.error("#"*100)
+                              @logger.error("# Error in checking modules ")
+                              @logger.error("# Please check if all modules are correctly installed, searched #{@modules.join(",")} but only detected #{modules_with_version.join(",")}")
+                              @logger.error("#"*100)
+                              "exit # Please check if all modules are correctly installed, searched #{@modules.join(",")} but only detected #{modules_with_version.join(",")}"
+                            end
                           else
                             ""
                           end
@@ -447,6 +458,13 @@ cd $SCRATCH_DIR || exit 1
   end
   def job_footer
     @out.print "#### JOB IS DONE WE PUT THINGS IN PLACE AND CLEAN AUP\n"
+    if File.exist?("/usr/local/ngseq/miniconda3/etc/profile.d/conda.sh")
+      @out.print <<-EOS
+. "/usr/local/ngseq/miniconda3/etc/profile.d/conda.sh"
+conda activate sushi
+      EOS
+    end
+
     if @output_files
       @output_files.map{|header| next_dataset[header]}.each do |file|
         # in actual case, to save under /srv/gstore/
@@ -759,28 +777,14 @@ rm -rf #{@scratch_dir} || exit 1
     if mock
       make_dummy_files
     end
-    copy_inputdataset_parameter_jobscripts
 
-    # job submittion
-    gstore_job_script_paths = []
-    @job_scripts.each_with_index do |job_script, i|
-      if job_id = submit(job_script, mock)
-        @job_ids << job_id
-        print "Submit job #{File.basename(job_script)} job_id=#{job_id}"
-        gstore_job_script_paths << File.join(@gstore_script_dir, File.basename(job_script))
-      end
-    end
-
-    puts
-    print 'job scripts: '
-    p @job_scripts
     print 'result dataset: '
     p @result_dataset
 
     # copy application data to gstore 
     @next_dataset_tsv_path = save_next_dataset_as_tsv
 
-    if !@job_ids.empty? and @dataset_sushi_id and dataset = DataSet.find_by_id(@dataset_sushi_id.to_i)
+    if @dataset_sushi_id and dataset = DataSet.find_by_id(@dataset_sushi_id.to_i)
       data_set_arr = []
       headers = []
       rows = []
@@ -805,20 +809,45 @@ rm -rf #{@scratch_dir} || exit 1
         unless @off_bfabric_registration
           if next_dataset = DataSet.find_by_id(@next_dataset_id)
             next_dataset.register_bfabric(bfabric_application_number: @next_dataset_bfabric_application_number)
+            if next_dataset.workunit_id
+              @job_scripts.each do |job_script|
+                open(job_script, "a") do |out|
+                  out.puts "WORKUNIT_ID=#{next_dataset.workunit_id}"
+                  out.puts "update_resource_size -w $WORKUNIT_ID"
+                end
+              end
+            end
           end
         end
+      end
+    end
+    copy_inputdataset_parameter_jobscripts
+
+    # job submittion
+    gstore_job_script_paths = []
+    @job_scripts.each_with_index do |job_script, i|
+      if job_id = submit(job_script, mock)
+        @job_ids << job_id
+        print "Submit job #{File.basename(job_script)} job_id=#{job_id}"
+        gstore_job_script_paths << File.join(@gstore_script_dir, File.basename(job_script))
+      end
+    end
+
+    puts
+    print 'job scripts: '
+    p @job_scripts
 
-        # save job and dataset relation in Sushi DB
-        job_ids.each_with_index do |job_id, i|
-          new_job = Job.new
-          new_job.submit_job_id = job_id.to_i
-          new_job.script_path = gstore_job_script_paths[i]
-          new_job.next_dataset_id = @next_dataset_id
-          new_job.save
-          new_job.data_set.jobs << new_job
-          new_job.data_set.save
-        end
 
+    unless @job_ids.empty? or NO_ROR
+      # save job and dataset relation in Sushi DB
+      job_ids.each_with_index do |job_id, i|
+        new_job = Job.new
+        new_job.submit_job_id = job_id.to_i
+        new_job.script_path = gstore_job_script_paths[i]
+        new_job.next_dataset_id = @next_dataset_id
+        new_job.save
+        new_job.data_set.jobs << new_job
+        new_job.data_set.save
       end
     end
     copy_nextdataset

data/lib/sushi_fabric/version.rb

@@ -1,3 +1,3 @@
 module SushiFabric
-  VERSION = "0.9.6"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sushi_fabric
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 1.0.1
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2020-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,8 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: workflow manager client.