rbbt-util 5.26.156 → 5.26.157

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 883e3e337b3a80b3ae8cdb0f31aad83733d32b99
-  data.tar.gz: 8b6fa24e110c55fe055e7c4ede4ae748525e7a1a
+SHA256:
+  metadata.gz: 4a7cb4caf03456a936ef3dab72d6ab957a2f3dce03324291fb10c0f4aabe527f
+  data.tar.gz: 8690112157a89e64313dc7fcec69c7192db507c210bac57cfd529271294b7174
 SHA512:
-  metadata.gz: c833d053a9706e594a70b6409161ab87e08b872fef9a3b3ad09770d4092470b906ebda2a0446731d2e5a8deba5e51d38a7b7145cb0bcdebf96c93899f6c0a56c
-  data.tar.gz: 7213a43f2a80ef6bfecff486efad61834e2835528ffe001e661fedb27de57e9d041c7652be8c72c411ff8a88ae919e44d915b78e032582150291d99212209417
+  metadata.gz: 68a347659e479c2cd5fdb09c15bbe09be216586c403f984689c6b72a360cb63f3c4815edabfb675362e4137440e67e5d364da836e4cca33c5ce200a5a7d19539
+  data.tar.gz: f77fa581d1b0cd8fd6d34f4ac63101a40162f0d7012e2ff9917c7f5b0de729a56f5c81a011528f676aac4cb59a30ebc09acdf47892bfadf1662d41a22ec2959c

data/lib/rbbt/resource.rb

@@ -93,7 +93,12 @@ module Resource
         timeout = 60 * 10
         Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
           http.request request do |response|
-            filename = response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
+            filename = if response["Content-Disposition"] 
+                         response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
+                       else
+                         nil
+                       end
+
             if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
               extension = filename.split(".").last
               final_path += '.' + extension

data/lib/rbbt/workflow.rb

@@ -598,6 +598,9 @@ module Workflow
   end
 
   def self.load_step(path)
+    Path.setup(path) unless Path === path
+    path = path.find
+
     begin
       _load_step(path)
     ensure

data/lib/rbbt/workflow/examples.rb

@@ -72,7 +72,16 @@ module Workflow
         end
 
       end
-      IndiferentHash.setup(inputs)
+      inputs = IndiferentHash.setup(inputs)
+
+      dir.glob("*#*").each do |od|
+        name = File.basename(od)
+        value = Open.read(od)
+        Log.debug "Loading override dependency #{ name } as #{value}"
+        inputs[name] = value.chomp
+      end
+
+      inputs
     end
   end
 

data/lib/rbbt/workflow/remote/remote_step.rb

@@ -258,7 +258,8 @@ class WorkflowRemoteClient
       stream = true if res.respond_to? :read
       join unless stream
       result_type ||= self.result_type
-      case result_type
+
+      case result_type.to_sym
       when :string
         stream ? res.read : res
       when :boolean
@@ -277,7 +278,6 @@ class WorkflowRemoteClient
         end
       when :array
         (stream ? res.read : res).split("\n")
-        res.split("\n")
       else
         json_text = if IO === res
                       res.read

data/lib/rbbt/workflow/remote/ssh/get.rb

@@ -1,4 +1,6 @@
 module WorkflowSSHClient
+  attr_accessor :override_dependencies
+
   def self.fix_hash(hash, fix_values = false)
     fixed = {}
     hash.each do |key, value|
@@ -107,7 +109,20 @@ module WorkflowSSHClient
     @input_id ||= "inputs-" << rand(100000).to_s
     @input_types = task_info(task)[:input_types]
 
-    WorkflowSSHClient.upload_inputs(@server, inputs, @input_types, @input_id)
+    if override_dependencies
+
+      if override_dependencies && override_dependencies.any?
+        override_dependencies.each do |od|
+          name, _sep, value = od.partition("=")
+          inputs[name] = value
+        end
+      end
+      iii inputs
+
+      WorkflowSSHClient.upload_inputs(@server, inputs, @input_types, @input_id)
+    else
+      WorkflowSSHClient.upload_inputs(@server, inputs, @input_types, @input_id)
+    end
 
     @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
       Misc.insist do
@@ -138,8 +153,13 @@ module WorkflowSSHClient
     end
   end
 
+  def load
+    load_res Open.open(path)
+  end
+
   def run(*args)
     produce(*args)
+    self.load unless args.first
   end
 
   def clean
@@ -148,4 +168,26 @@ module WorkflowSSHClient
     _restart
   end
 
+  def self.relay(workflow, task, jobname, inputs, server, options = {})
+    options = Misc.add_defaults options, :search_path => 'user'
+    search_path = options[:search_path]
+
+    job = workflow.job(task, jobname, inputs)
+
+    job.dependencies.each do |dep| 
+      dep.produce 
+    end
+
+    override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+
+    job.dependencies.each do |dep| 
+      Step.migrate(dep.path, search_path, :target => server)
+    end
+
+    remote = WorkflowRemoteClient.new("ssh://#{server}:#{workflow.to_s}", "#{workflow.to_s}")
+    rjob = remote.job(task, jobname, {})
+    rjob.override_dependencies = override_dependencies
+    rjob.run
+  end
+
 end

data/lib/rbbt/workflow/step/accessor.rb

@@ -77,7 +77,8 @@ class Step
 
   def self.save_inputs(inputs, input_types, dir)
     inputs.each do |name,value|
-      type = input_types[name].to_s
+      type = input_types[name]
+      type = type.to_s if type
       path = File.join(dir, name.to_s)
 
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
@@ -115,6 +116,11 @@ class Step
       next if value.nil?
       inputs[name] = value
     end
+
+    if options.include? 'override_dependencies'
+      inputs.merge!(:override_dependencies => open[:override_dependencies])
+      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
+    end
     save_inputs(inputs, input_types, dir)
 
     inputs.any?

data/lib/rbbt/workflow/step/run.rb

@@ -290,7 +290,7 @@ class Step
 
           @inputs = new_inputs if @inputs
 
-          if not task.inputs.nil?
+          if @inputs && ! task.inputs.nil?
             info_inputs = @inputs.collect do |i| 
               if Path === i 
                 i.to_s
@@ -361,7 +361,8 @@ class Step
                     merge_info({
                       :done => (done_time = Time.now),
                       :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
-                      :time_elapsed => (time_elapsed = done_time - start_time)
+                      :time_elapsed => (time_elapsed = done_time - start_time),
+                      :versions => Rbbt.versions
                     })
                     log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
                   end
@@ -402,7 +403,8 @@ class Step
             merge_info({
               :done => (done_time = Time.now),
               :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
-              :time_elapsed => (time_elapsed = done_time - start_time)
+              :time_elapsed => (time_elapsed = done_time - start_time),
+              :versions => Rbbt.versions
             })
             log :ending
             Step.purge_stream_cache
@@ -420,7 +422,6 @@ class Step
           result
         end # END PERSIST
         log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
-        set_info :versions, Rbbt.versions
 
         if no_load
           @result ||= result

data/lib/rbbt/workflow/util/archive.rb

@@ -1,5 +1,8 @@
 require 'rbbt/workflow/remote/ssh/driver'
 class Step
+
+  MAIN_RSYNC_ARGS="-avztAXHP"
+  
   def self.link_job(path, target_dir, task = nil, workflow = nil)
     Path.setup(target_dir)
 
@@ -40,7 +43,7 @@ class Step
 
       Misc.in_dir(tmpdir) do
         if File.directory?(target)
-          CMD.cmd_log("rsync -avztAXHP --copy-unsafe-links '#{ tmpdir }/' '#{ target }/'")
+          CMD.cmd_log("rsync #{MAIN_RSYNC_ARGS} --copy-unsafe-links '#{ tmpdir }/' '#{ target }/'")
         else
           CMD.cmd_log("tar cvhzf '#{target}'  ./*")
         end
@@ -111,7 +114,7 @@ class Step
 
       Misc.in_dir(tmpdir) do
         if File.directory?(target)
-          CMD.cmd_log("rsync -avztAXHP --copy-unsafe-links '#{ tmpdir }/' '#{ target }/'")
+          CMD.cmd_log("rsync #{MAIN_RSYNC_ARGS} --copy-unsafe-links '#{ tmpdir }/' '#{ target }/'")
         else
           CMD.cmd_log("tar cvhzf '#{target}'  ./*")
         end
@@ -214,7 +217,7 @@ puts resource[path].find(search_path)
       TmpFile.with_file(files_and_dirs.sort_by{|l| l.length}.to_a * "\n") do |tmp_include_file|
         test_str = options[:test] ? '-nv' : ''
 
-        cmd = "rsync -avztAXHP --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
+        cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
 
         cmd << " && rm -Rf #{source}" if options[:delete]
 

data/share/rbbt_commands/workflow/task

@@ -73,7 +73,12 @@ def fix_options(workflow, task, job_options)
   job_options_cleaned = {}
 
   job_options.each do |name, value|
-    value = case input_types[name].to_sym
+    type = input_types[name]
+    type = type.to_sym if type
+
+    value = case type
+            when nil
+                value
             when :boolean
               TrueClass === value or %w(true TRUE T yes).include? value
             when :float
@@ -94,7 +99,7 @@ def fix_options(workflow, task, job_options)
                 end
               end
             when :array
-              if input_options[name] and input_options[name][:stream] and String === value and Misc.is_filename?(value) and not input_options[name][:nofile]
+              if input_options[name] && input_options[name][:stream] && String === value && Misc.is_filename?(value) && !! input_options[name][:nofile]
                 get_value_stream(value)
               elsif input_options[name] and input_options[name][:stream] and value == "-"
                 STDIN
@@ -394,7 +399,7 @@ begin
 
   if tasks = options.delete(:prepare)
     tasks = tasks.split(",")
-    prepare_cpus = (options[:prepare_cpus] || 1)
+    prepare_cpus = (options[:prepare_cpus] || 1)
     puts Step.prepare_dependencies(job, tasks,  prepare_cpus)
     exit 0
   end

data/test/rbbt/workflow/remote/test_client.rb

@@ -18,27 +18,26 @@ end
 
 class TestRemote < Test::Unit::TestCase
 
-  def _test_ssh
+  def test_ssh
     Log.severity = 0
-    client = WorkflowRemoteClient.new "ssh://localhost:Translation", "Translation"
+    client = WorkflowRemoteClient.new "ssh://turbo:Translation", "Translation"
     job = client.job("translate", "SSH-TEST", :genes => ["TP53","KRAS"])
-    iii job.url
-    puts job.run
+    assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
   end
 
   def test_rest
     Log.with_severity 0 do
 
       workflow_server(TestWFRest) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run
+        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
+        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
+        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
       end
 
       workflow_server(TestWFRest, :Port => 1902) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run
+        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
+        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
+        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.156
+  version: 5.26.157
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-23 00:00:00.000000000 Z
+date: 2020-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -537,94 +537,93 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/test_association.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/test_entity.rb
+- test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_step.rb
+- test/rbbt/workflow/remote/test_client.rb
+- test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/resource/test_path.rb
+- test/rbbt/util/test_colorize.rb
 - test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/misc/test_bgzf.rb
-- test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_pipes.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/test_semaphore.rb
-- test/rbbt/util/test_config.rb
-- test/rbbt/util/test_R.rb
+- test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/R/test_plot.rb
 - test/rbbt/util/R/test_eval.rb
 - test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_plot.rb
-- test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_config.rb
 - test/rbbt/util/test_log.rb
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/test_tmpfile.rb
+- test/rbbt/util/test_R.rb
+- test/rbbt/util/test_excel2tsv.rb
 - test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/test_python.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_filecache.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_fix_width_table.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/tsv/test_change_id.rb
 - test/rbbt/tsv/test_attach.rb
-- test/rbbt/tsv/test_stream.rb
-- test/rbbt/tsv/test_excel.rb
-- test/rbbt/tsv/parallel/test_through.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/test_matrix.rb
-- test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_field_index.rb
+- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_parallel.rb
 - test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_parser.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/test_packed_index.rb
+- test/rbbt/tsv/test_excel.rb
+- test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_stream.rb
+- test/rbbt/test_association.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb
 - test/rbbt/test_annotations.rb
-- test/rbbt/test_persist.rb
+- test/rbbt/test_fix_width_table.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/entity/test_identifiers.rb
 - test/rbbt/annotations/test_util.rb
-- test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/test_step.rb
-- test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_task.rb
-- test/rbbt/workflow/remote/test_client.rb
+- test/rbbt/test_hpc.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_registry.rb
 - test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_tsv.rb
-- test/rbbt/test_hpc.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/test_entity.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/persist/test_tsv.rb
 - test/test_helper.rb