sushi_fabric 0.9.3 → 0.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 84de00c578f9193950709558ab94634ef637a12e
-  data.tar.gz: fb3af7878169c568fdcf8d291dcc354ef3fa5142
+SHA256:
+  metadata.gz: c57dc465158ddf76e857e126e427f581ecf1a65a805fd01af17e1782b0503208
+  data.tar.gz: 7ff6d22fd4617f2e10459474d86f706b1f4bbd39c7922031271c49c8f4f45269
 SHA512:
-  metadata.gz: 7e83696b93bbc02318541f5e7be8fe406d86ab2330fc28b2a1ec3b256a568bff3b4e3771b925a83adceec0c6984ff0f842d4a924f94b52ee46b4c34aec558128
-  data.tar.gz: 32e32f6a899a3e1d435da76c945a3c76835ff8cc0fb02b8b484fc36333dc5b03b02be9a5f7f14cfaaba56fcab1e58e638fa2e2478f036c685fed9b456cbf6195
+  metadata.gz: c88d5589e019b1742b962d78550658c53302d453a2ce76c21b171f9caa4cc176be79ccb60dbb3d9edfb9cc88e7fbde1a36848a8392a64f089b1ffcbb53a99260
+  data.tar.gz: eacc4d03b02f4a94adeb42a40f2217022aa0b7659c1cfe3fc1035d9726dd04375e15eebf47c4a7f582416c7c7b18eabaac4e9c8b6e46c5c48fbea4139380ada6

data/lib/sushi_fabric/sushiApp.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
-# Version = '20191114-151205'
+# Version = '20200723-132451'
 
 require 'csv'
 require 'fileutils'
@@ -227,6 +227,7 @@ class SushiApp
   attr_accessor :mango_run_name
   attr_accessor :input_dataset_bfabric_application_number
   attr_accessor :next_dataset_bfabric_application_number
+  attr_reader :inactivate_nodes
   def initialize
     @gstore_dir = GSTORE_DIR
     @project = nil
@@ -236,6 +237,7 @@ class SushiApp
     @params['ram'] = nil
     @params['scratch'] = nil
     @params['node'] = ''
+    @params['queue'] = ''
     @params['process_mode'] = 'SAMPLE'
     @params['samples'] = ''
     @job_ids = []
@@ -243,6 +245,7 @@ class SushiApp
     @module_source = MODULE_SOURCE
     @modules = []
     #@workflow_manager = workflow_manager_instance||DRbObject.new_with_uri(WORKFLOW_MANAGER)
+    @last_job = true
   end
   def set_input_dataset
     if @dataset_tsv_file
@@ -389,8 +392,13 @@ class SushiApp
   end
   def check_latest_module_version(mod)
     command_out =  %x[ bash -lc "source #{@module_source}; module whatis #{mod} 2>&1" ]
-    latest_mod = command_out.split.first
-    latest_mod = nil if latest_mod == "Failed"
+    latest_mod = nil
+    command_out.split("\n").each do |line|
+      if line =~ /#{mod}/
+        latest_mod = line.split.first
+        break
+      end
+    end
     latest_mod
   end
   def job_header
@@ -414,7 +422,6 @@ class SushiApp
                             modules_with_version = @modules.map{|mod| check_latest_module_version(mod)}
                             modules_with_version.compact!
                             "module add #{modules_with_version.join(' ')}"
-                            #"module add #{@modules.join(' ')}"
                           else
                             ""
                           end
@@ -429,6 +436,7 @@ umask 0002
 SCRATCH_DIR=#{@scratch_dir}
 GSTORE_DIR=#{@gstore_dir}
 INPUT_DATASET=#{@input_dataset_tsv_path}
+LAST_JOB=#{@last_job.to_s.upcase}
 echo "Job runs on `hostname`"
 echo "at $SCRATCH_DIR"
 mkdir $SCRATCH_DIR || exit 1
@@ -631,36 +639,22 @@ rm -rf #{@scratch_dir} || exit 1
     @out.close
   end
   def sample_mode
-    set_job_script =-> (dataset_) do
-        @dataset = dataset_
+    selected_samples = Hash[*@params['samples'].split(',').map{|sample_name| [sample_name, true]}.flatten]
+    @dataset_hash.each_with_index do |row, i|
+      @dataset = Hash[*row.map{|key,value| [key.gsub(/\[.+\]/,'').strip, value]}.flatten]
+      if selected_samples[@dataset['Name']]
         sample_name = @dataset['Name']||@dataset.first
         @job_script = if @dataset_sushi_id and dataset = DataSet.find_by_id(@dataset_sushi_id.to_i)
                         File.join(@job_script_dir, @analysis_category + '_' + sample_name) + '_' + dataset.name.gsub(/\s+/,'_') + '.sh'
                       else
                         File.join(@job_script_dir, @analysis_category + '_' + sample_name) + '.sh'
                       end
+        @last_job = (i == @dataset_hash.length - 1)
         make_job_script
         @job_scripts << @job_script
         @result_dataset << next_dataset
-    end
-    selected_samples = Hash[*@params['samples'].split(',').map{|sample_name| [sample_name, true]}.flatten]
-    last_sample = nil
-    @dataset_hash.each do |row|
-      @dataset = Hash[*row.map{|key,value| [key.gsub(/\[.+\]/,'').strip, value]}.flatten]
-      if selected_samples[@dataset['Name']]
-        header = if header_ = row.keys.find{|h| h.tag?('Last')}
-                   header_.gsub(/\[.+\]/,'').strip
-                 end
-        if header and eval(@dataset[header])
-          last_sample = @dataset
-        else
-          set_job_script.(@dataset)
-        end
       end
     end
-    if last_sample
-      set_job_script.(last_sample)
-    end
   end
   def dataset_mode
     selected_samples = Hash[*@params['samples'].split(',').map{|sample_name| [sample_name, true]}.flatten]
@@ -766,28 +760,14 @@ rm -rf #{@scratch_dir} || exit 1
     if mock
       make_dummy_files
     end
-    copy_inputdataset_parameter_jobscripts
-
-    # job submittion
-    gstore_job_script_paths = []
-    @job_scripts.each_with_index do |job_script, i|
-      if job_id = submit(job_script, mock)
-        @job_ids << job_id
-        print "Submit job #{File.basename(job_script)} job_id=#{job_id}"
-        gstore_job_script_paths << File.join(@gstore_script_dir, File.basename(job_script))
-      end
-    end
 
-    puts
-    print 'job scripts: '
-    p @job_scripts
     print 'result dataset: '
     p @result_dataset
 
     # copy application data to gstore 
     @next_dataset_tsv_path = save_next_dataset_as_tsv
 
-    if !@job_ids.empty? and @dataset_sushi_id and dataset = DataSet.find_by_id(@dataset_sushi_id.to_i)
+    if @dataset_sushi_id and dataset = DataSet.find_by_id(@dataset_sushi_id.to_i)
       data_set_arr = []
       headers = []
       rows = []
@@ -812,20 +792,46 @@ rm -rf #{@scratch_dir} || exit 1
         unless @off_bfabric_registration
           if next_dataset = DataSet.find_by_id(@next_dataset_id)
             next_dataset.register_bfabric(bfabric_application_number: @next_dataset_bfabric_application_number)
+            if next_dataset.workunit_id
+              @job_scripts.each do |job_script|
+                open(job_script, "a") do |out|
+                  out.puts "module load Dev/Python"
+                  out.puts "WORKUNIT_ID=#{next_dataset.workunit_id}"
+                  out.puts "update_resource_size -w $WORKUNIT_ID"
+                end
+              end
+            end
           end
         end
+      end
+    end
+    copy_inputdataset_parameter_jobscripts
+
+    # job submittion
+    gstore_job_script_paths = []
+    @job_scripts.each_with_index do |job_script, i|
+      if job_id = submit(job_script, mock)
+        @job_ids << job_id
+        print "Submit job #{File.basename(job_script)} job_id=#{job_id}"
+        gstore_job_script_paths << File.join(@gstore_script_dir, File.basename(job_script))
+      end
+    end
+
+    puts
+    print 'job scripts: '
+    p @job_scripts
 
-        # save job and dataset relation in Sushi DB
-        job_ids.each_with_index do |job_id, i|
-          new_job = Job.new
-          new_job.submit_job_id = job_id.to_i
-          new_job.script_path = gstore_job_script_paths[i]
-          new_job.next_dataset_id = @next_dataset_id
-          new_job.save
-          new_job.data_set.jobs << new_job
-          new_job.data_set.save
-        end
 
+    unless @job_ids.empty? or NO_ROR
+      # save job and dataset relation in Sushi DB
+      job_ids.each_with_index do |job_id, i|
+        new_job = Job.new
+        new_job.submit_job_id = job_id.to_i
+        new_job.script_path = gstore_job_script_paths[i]
+        new_job.next_dataset_id = @next_dataset_id
+        new_job.save
+        new_job.data_set.jobs << new_job
+        new_job.data_set.save
       end
     end
     copy_nextdataset

data/lib/sushi_fabric/version.rb

@@ -1,3 +1,3 @@
 module SushiFabric
-  VERSION = "0.9.3"
+  VERSION = "0.9.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sushi_fabric
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.8
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-14 00:00:00.000000000 Z
+date: 2020-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,8 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: workflow manager client.