sushi_fabric 1.1.1 → 1.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f27bf481942df3066dc10ead974eb849cbc0c2f7ad010599d2d75b7cdf9d6762
-  data.tar.gz: fd39ea3911fd3f9571ceb4c3b2071e51103109d85f7e9e042edabddc95733adc
+  metadata.gz: df087d665373defd52bfde8eb52acd6f3bfafd3092d220f7c95cdc75d949045c
+  data.tar.gz: 4dbc0933dfe40e740879b2a7b757f75a16f1a0f88a8df67800537c1ea3d42ea0
 SHA512:
-  metadata.gz: 36560e6365fe052621295480761c71b1082601f645261a728910e8c03372a601fdfd8ea9883233a68212eaf0cbd60598a32c62fcbc068445b6c150bdf170aeb8
-  data.tar.gz: 930e64f997d8383991e88c5c6f21fab27d5c1f40e59e16987b002a62615d96aa65571eca1c055708c75b0f96496bd3bf3a9ee72d025e5a75e1e1e79b8143c973
+  metadata.gz: dfa9b02cd70f2b112d2ffc9c3679a7d1590a5ce9388149588b7d026d34b705832af3371b7c84f9a873056dda454372be9dfd33e8cb491ddc7322b0f6192402a7
+  data.tar.gz: 4b79c6dda54fb542ac3d4e0bad03f5333729b76f21b0e2758b14a3402b6dba04f9824b1e26d9cca05e050a69af19cf5e3ce8e8338894b116c4ad89a61144419d

data/lib/sushi_fabric/sushiApp.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
-# Version = '20210813-192118'
+# Version = '20211209-162918'
 
 require 'csv'
 require 'fileutils'
@@ -20,6 +20,7 @@ module SushiFabric
     config.scratch_dir = '/tmp/scratch'
     config.module_source = nil
     config.course_mode = nil
+    config.rails_host = nil
   end
 
   # load custmized parameters if there is
@@ -41,6 +42,7 @@ module SushiFabric
     config.scratch_dir = '/tmp/scratch'
     config.module_source = nil
     config.course_mode = nil
+    config.rails_host = nil
   end
 end
       EOF
@@ -53,7 +55,7 @@ end
   SUSHI_APP_DIR = config.sushi_app_dir
   SCRATCH_DIR = config.scratch_dir
   MODULE_SOURCE = config.module_source
-  
+  RAILS_HOST = config.rails_host 
   unless File.exist?(GSTORE_DIR)
     FileUtils.mkdir_p GSTORE_DIR
   end
@@ -238,8 +240,6 @@ class SushiApp
     @params['cores'] = nil
     @params['ram'] = nil
     @params['scratch'] = nil
-    @params['node'] = ''
-    @params['queue'] = ''
     @params['partition'] = ''
     @params['process_mode'] = 'SAMPLE'
     @params['samples'] = ''
@@ -384,14 +384,23 @@ class SushiApp
     @result_dir = File.join(@project, @result_dir_base)
     @scratch_result_dir = File.join(SCRATCH_DIR, @result_dir_base)
     @job_script_dir = File.join(@scratch_result_dir, 'scripts')
+    @uploaded_files_dir = File.join(@scratch_result_dir, 'uploaded')
     @gstore_result_dir = File.join(@gstore_dir, @result_dir)
     @gstore_script_dir = File.join(@gstore_result_dir, 'scripts')
     @gstore_project_dir = File.join(@gstore_dir, @project)
+    @gstore_uploaded_dir = File.join(@gstore_result_dir, 'uploaded')
     set_file_paths
   end
   def prepare_result_dir
     FileUtils.mkdir_p(@scratch_result_dir)
     FileUtils.mkdir_p(@job_script_dir)
+    @uploaded_files = []
+    @params.each do |key, value|
+      if @params[key, 'file_upload']
+        FileUtils.mkdir_p(@uploaded_files_dir)
+        @uploaded_files << value
+      end
+    end
   end
   def check_latest_modules_version(modules)
     command_out =  %x[ bash -lc "source #{@module_source}; module whatis #{modules.join(" ")} 2>&1" | cut -f 1 -d " " | uniq ]
@@ -515,6 +524,7 @@ rm -rf #{@scratch_dir} || exit 1
     gsub_options << "-p #{@params['partition']}" unless @params['partition'].to_s.empty?
     gsub_options << "-r #{@params['ram']}" unless @params['ram'].to_s.empty?
     gsub_options << "-s #{@params['scratch']}" unless @params['scratch'].to_s.empty?
+    gsub_options << "-i #{@params['nice']}" unless @params['nice'].to_s.empty?
     command = "wfm_monitoring --server #{WORKFLOW_MANAGER} --user #{@user} --project #{@project.gsub(/p/,'')} --logdir #{@gstore_script_dir} #{job_script} #{gsub_options.join(' ')}"
     puts "submit: #{command}"
 
@@ -524,7 +534,7 @@ rm -rf #{@scratch_dir} || exit 1
     job_id = 0
     begin
       #job_id = @workflow_manager.start_monitoring(job_script, @user, 0, script_content, project_number, gsub_options.join(' '), @gstore_script_dir)
-      job_id = @workflow_manager.start_monitoring3(job_script, script_content, @user, project_number, gsub_options.join(' '), @gstore_script_dir, @next_dataset_id)
+      job_id = @workflow_manager.start_monitoring3(job_script, script_content, @user, project_number, gsub_options.join(' '), @gstore_script_dir, @next_dataset_id, RAILS_HOST)
     rescue => e
       time = Time.now.strftime("[%Y.%m.%d %H:%M:%S]")
       @logger.error("*"*50)
@@ -576,7 +586,13 @@ rm -rf #{@scratch_dir} || exit 1
     CSV.open(file_path, 'w', :col_sep=>"\t") do |out|
       out << ["sushi_app", self.class.name]
       @output_params.each do |key, value|
-        out << [key, value]
+        if @output_params[key, 'file_upload']
+          uploaded_file_path = File.join(@result_dir, "uploaded", File.basename(value))
+          out << [key, uploaded_file_path]
+          @params[key] = uploaded_file_path
+        else
+          out << [key, value]
+        end
       end
     end
     file_path
@@ -628,6 +644,18 @@ rm -rf #{@scratch_dir} || exit 1
     end
     com
   end
+  def copy_uploaded_files
+    if not @uploaded_files.empty?
+      @uploaded_files.each do |file|
+        FileUtils.cp(file, @uploaded_files_dir)
+        command = "cp #{file} #{@uploaded_files_dir}"
+        puts command
+        FileUtils.rm_r(File.dirname(file))
+        command = "rm -rf #{File.dirname(file)}"
+        puts command
+      end
+    end
+  end
   def copy_inputdataset_parameter_jobscripts
     org = @scratch_result_dir
     dest = @gstore_project_dir
@@ -840,6 +868,7 @@ rm -rf #{@scratch_dir} || exit 1
         end
       end
     end
+    copy_uploaded_files
     copy_inputdataset_parameter_jobscripts
 
     # job submittion

data/lib/sushi_fabric/version.rb

@@ -1,3 +1,3 @@
 module SushiFabric
-  VERSION = "1.1.1"
+  VERSION = "1.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sushi_fabric
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.5
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler