sushi_fabric 1.1.0 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e8c7afdc3db914867cb23a1274fe257e8d16f60904c867e84ada16b3faeec5b
-  data.tar.gz: 9e0db5e3421c31767b1c089652831dee04fd536e557969ad17519dbd1f681d0a
+  metadata.gz: b34ce73bf64c4447679ec3fa06820fccf59857ea739a9b16e9a91907051c1894
+  data.tar.gz: 5d7f45f91531ecb9c7014b6b3694622694b6257845b607cec8819256cbff9a2f
 SHA512:
-  metadata.gz: ff6aa500c3ce2f3bdf1a33d3829b1aed9031c949fc894f69d23b9cbb7e47b4fbdb5c4477d3cb916e139c944521eeae0b619e7f1cc1df1991054d1b94cf7dd528
-  data.tar.gz: a491d23bef554e292253aa28f08677b940bf8f2fd6a6d798a7423a1a62fba439bb378321ceb99464fd68a044f6031a3e25c578387406951a1c2f5b18e797196f
+  metadata.gz: 190e59e9c16c3e1031fdc929e6c4a3627cd63d0e1b6ecaf95aeba88158a08af587a398c9903efbcaf3efdc2cafafd39f015a3714365e4b7ed45adcdb18b83a66
+  data.tar.gz: 5ee4d5c6a57b06fedb6cecdf0cc396376846e1fe1ed41d431f3104ab140e0bfd0dbb34cb4df144e0cd08390315355498e09228509ded24ab976466d155358c80

data/lib/sushi_fabric/sushiApp.rb

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
-# Version = '20210730-151826'
+# Version = '20211207-151042'
 
 require 'csv'
 require 'fileutils'
@@ -20,6 +20,7 @@ module SushiFabric
     config.scratch_dir = '/tmp/scratch'
     config.module_source = nil
     config.course_mode = nil
+    config.rails_host = nil
   end
 
   # load custmized parameters if there is
@@ -41,6 +42,7 @@ module SushiFabric
     config.scratch_dir = '/tmp/scratch'
     config.module_source = nil
     config.course_mode = nil
+    config.rails_host = nil
   end
 end
       EOF
@@ -53,7 +55,7 @@ end
   SUSHI_APP_DIR = config.sushi_app_dir
   SCRATCH_DIR = config.scratch_dir
   MODULE_SOURCE = config.module_source
-  
+  RAILS_HOST = config.rails_host 
   unless File.exist?(GSTORE_DIR)
     FileUtils.mkdir_p GSTORE_DIR
   end
@@ -238,8 +240,6 @@ class SushiApp
     @params['cores'] = nil
     @params['ram'] = nil
     @params['scratch'] = nil
-    @params['node'] = ''
-    @params['queue'] = ''
     @params['partition'] = ''
     @params['process_mode'] = 'SAMPLE'
     @params['samples'] = ''
@@ -384,14 +384,23 @@ class SushiApp
     @result_dir = File.join(@project, @result_dir_base)
     @scratch_result_dir = File.join(SCRATCH_DIR, @result_dir_base)
     @job_script_dir = File.join(@scratch_result_dir, 'scripts')
+    @uploaded_files_dir = File.join(@scratch_result_dir, 'uploaded')
     @gstore_result_dir = File.join(@gstore_dir, @result_dir)
     @gstore_script_dir = File.join(@gstore_result_dir, 'scripts')
     @gstore_project_dir = File.join(@gstore_dir, @project)
+    @gstore_uploaded_dir = File.join(@gstore_result_dir, 'uploaded')
     set_file_paths
   end
   def prepare_result_dir
     FileUtils.mkdir_p(@scratch_result_dir)
     FileUtils.mkdir_p(@job_script_dir)
+    @uploaded_files = []
+    @params.each do |key, value|
+      if @params[key, 'file_upload']
+        FileUtils.mkdir_p(@uploaded_files_dir)
+        @uploaded_files << value
+      end
+    end
   end
   def check_latest_modules_version(modules)
     command_out =  %x[ bash -lc "source #{@module_source}; module whatis #{modules.join(" ")} 2>&1" | cut -f 1 -d " " | uniq ]
@@ -515,6 +524,7 @@ rm -rf #{@scratch_dir} || exit 1
     gsub_options << "-p #{@params['partition']}" unless @params['partition'].to_s.empty?
     gsub_options << "-r #{@params['ram']}" unless @params['ram'].to_s.empty?
     gsub_options << "-s #{@params['scratch']}" unless @params['scratch'].to_s.empty?
+    gsub_options << "-i #{@params['nice']}" unless @params['nice'].to_s.empty?
     command = "wfm_monitoring --server #{WORKFLOW_MANAGER} --user #{@user} --project #{@project.gsub(/p/,'')} --logdir #{@gstore_script_dir} #{job_script} #{gsub_options.join(' ')}"
     puts "submit: #{command}"
 
@@ -524,7 +534,7 @@ rm -rf #{@scratch_dir} || exit 1
     job_id = 0
     begin
       #job_id = @workflow_manager.start_monitoring(job_script, @user, 0, script_content, project_number, gsub_options.join(' '), @gstore_script_dir)
-      job_id = @workflow_manager.start_monitoring3(job_script, script_content, @user, project_number, gsub_options.join(' '), @gstore_script_dir)
+      job_id = @workflow_manager.start_monitoring3(job_script, script_content, @user, project_number, gsub_options.join(' '), @gstore_script_dir, @next_dataset_id, RAILS_HOST)
     rescue => e
       time = Time.now.strftime("[%Y.%m.%d %H:%M:%S]")
       @logger.error("*"*50)
@@ -576,7 +586,12 @@ rm -rf #{@scratch_dir} || exit 1
     CSV.open(file_path, 'w', :col_sep=>"\t") do |out|
       out << ["sushi_app", self.class.name]
       @output_params.each do |key, value|
-        out << [key, value]
+        if @output_params[key, 'file_upload']
+          uploaded_file_path = File.join(@gstore_uploaded_dir, File.basename(value))
+          out << [key, uploaded_file_path]
+        else
+          out << [key, value]
+        end
       end
     end
     file_path
@@ -628,6 +643,18 @@ rm -rf #{@scratch_dir} || exit 1
     end
     com
   end
+  def copy_uploaded_files
+    if not @uploaded_files.empty?
+      @uploaded_files.each do |file|
+        FileUtils.cp(file, @uploaded_files_dir)
+        command = "cp #{file} #{@uploaded_files_dir}"
+        puts command
+        FileUtils.rm_r(File.dirname(file))
+        command = "rm -rf #{File.dirname(file)}"
+        puts command
+      end
+    end
+  end
   def copy_inputdataset_parameter_jobscripts
     org = @scratch_result_dir
     dest = @gstore_project_dir
@@ -840,6 +867,7 @@ rm -rf #{@scratch_dir} || exit 1
         end
       end
     end
+    copy_uploaded_files
     copy_inputdataset_parameter_jobscripts
 
     # job submittion

data/lib/sushi_fabric/version.rb

@@ -1,3 +1,3 @@
 module SushiFabric
-  VERSION = "1.1.0"
+  VERSION = "1.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sushi_fabric
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.4
 platform: ruby
 authors:
 - Functional Genomics Center Zurich
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-30 00:00:00.000000000 Z
+date: 2021-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler