rbbt-util 5.37.16 → 5.38.1

data/share/rbbt_commands/workflow/task

@@ -93,6 +93,8 @@ def fix_options(workflow, task, job_options)
               TrueClass === value or %w(true TRUE T yes).include? value
             when :float
               value.to_f
+            when :path
+              Path.setup(value)
             when :integer
               value.to_i
             when :text
@@ -419,13 +421,6 @@ begin
     exit 0
   end
 
-  if tasks = options.delete(:produce)
-    tasks = tasks.split(",")
-    produce_cpus = (options[:produce_cpus] || 1)
-    puts Step.produce_dependencies(job, tasks,  prepare_cpus)
-    exit 0
-  end
-
   def match_dependencies(queries, dependencies)
     queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
 
@@ -443,30 +438,69 @@ begin
     matched
   end
 
+  def replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay = false, run_type = :run)
+    jobs_to_relay.each do |job|
+      ComputeDependency.setup(job, :bootstrap)
+      next if job.done?
+      Log.low "Relaying #{Misc.fingerprint job} to #{server}"
+      jmeta = class << job; self; end
+
+      job.instance_variable_set(:@job, job)
+      job.instance_variable_set(:@host, server)
+      job.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
+
+      jmeta.define_method :run do |*args|
+        if done?
+          load
+        else
+          RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
+          Step.migrate(@job, 'user', :source => @host)
+          load
+        end
+      end
+      #job.dependencies = []
+
+      #([job] + job.rec_dependencies).each do |j|
+      #  next if job.done?
+      #  jmeta = class << j; self; end
+
+      #  j.instance_variable_set(:@job, job)
+      #  j.instance_variable_set(:@host, server)
+      #  j.instance_variable_set(:@produce_dependencies, produce_dependencies_for_relay)
+
+      #  jmeta.define_method :run do |*args|
+      #    if done?
+      #      load
+      #    else
+      #      RemoteWorkflow::SSH.relay_job_list([@job], @host, :run_type => run_type, :migrate => true, :produce_dependencies => @produce_dependencies)
+      #      Step.migrate(@job, 'user', :source => @host)
+      #      load
+      #    end
+      #  end
+      #end
+    end
+  end
+
+
   if server = options.delete(:relay)
     require 'rbbt/workflow/remote_workflow'
     relay_dependencies = options.delete(:relay_dependencies).split(",")
     produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
-    if relay_dependencies
-      match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
-        RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
-      end
-    else
-      RemoteWorkflow::SSH.relay_job(job, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
-    end
+
+    jobs_to_relay = relay_dependencies ?  match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
+    jobs_to_relay.reject!{|d| d.done? }
+
+    replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :run)
   end
 
   if server = options.delete(:slurm_relay)
     require 'rbbt/workflow/remote_workflow'
     relay_dependencies = options.delete(:relay_dependencies).split(",")
     produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
-    if relay_dependencies
-      match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
-        RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
-      end
-    else
-      RemoteWorkflow::SSH.relay_job(job, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
-    end
+    jobs_to_relay = relay_dependencies ?  match_dependencies(relay_dependencies, job.rec_dependencies) : [job]
+    jobs_to_relay.reject!{|d| d.done? }
+
+    replace_relayed_jobs(jobs_to_relay, server, produce_dependencies_for_relay, :orchestrate)
   end
 
 
@@ -488,6 +522,14 @@ begin
     end
   end
 
+  if tasks = options.delete(:produce)
+    tasks = tasks.split(",")
+    produce_cpus = (options[:produce_cpus] || 1)
+    puts Step.produce_dependencies(job, tasks,  produce_cpus)
+    exit 0
+  end
+
+
   if do_fork
     ENV["RBBT_NO_PROGRESS"] = "true"
     if detach

data/test/rbbt/test_resource.rb

@@ -7,6 +7,8 @@ require 'test/unit'
 module TestResource
   extend Resource
 
+  self.subdir = 'tmp/tmp-share/resource/TestResource/'
+
   claim tmp.test.google, :url, "http://google.com"
   claim tmp.test.string, :string, "TEST"
   claim tmp.test.proc, :proc do
@@ -79,8 +81,8 @@ class TestTSV < Test::Unit::TestCase
   end
 
   def test_libdir
-    assert File.exist?(TestResource[].share.Rlib["util.R"].find(:lib))
-    assert File.exist?(TestResource[].share.Rlib["util.R"].find)
+    assert File.exist?(Rbbt.share.Rlib["util.R"].find(:lib))
+    assert File.exist?(Rbbt.share.Rlib["util.R"].find)
   end
 
   def __test_server
@@ -100,6 +102,9 @@ class TestTSV < Test::Unit::TestCase
     assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
     assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/share/databases/DATABASE/FILE'))
     assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
+
+    assert_equal 'etc/somefile', TestResource.identify(TestResource.etc.somefile.find(:user))
+    assert_equal 'etc/somefile', TestResource.identify(TestResource.etc.somefile.find(:local))
   end
 
 end

data/test/rbbt/tsv/test_accessor.rb

@@ -199,13 +199,30 @@ row3    A|AA|AAA|AAA    B    Id3    3
 
   def test_unzip
     content =<<-EOF
-#Id    ValueA    ValueB    OtherID
-row1    a|A    b|B    Id1|Id2
-row2    aa|aa|AA|AA    b1|b2|B1|B2    Id1|Id1|Id2|Id2
+#: :type=:list
+#PMID:Sentence number:TF:TG	Transcription Factor (Associated Gene Name)	Target Gene (Associated Gene Name)	Sign	Negation	PMID
+24265317:3:NR1H3:FASN	NR1H3	FASN			24265317
+17522048:0:NR1H3:FASN	NR1H3	FASN	+		17522048
+19903962:0:NR1H3:FASN	NR1H3	FASN			19903962
+19903962:7:NR1H3:FASN	NR1H3	FASN			19903962
+22183856:4:NR1H3:FASN	NR1H3	FASN			22183856
+22641099:4:NR1H3:FASN	NR1H3	FASN	+		22641099
+23499676:8:NR1H3:FASN	NR1H3	FASN	+		23499676
+11790787:5:NR1H3:FASN	NR1H3	FASN			11790787
+11790787:7:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:9:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:11:NR1H3:FASN	NR1H3	FASN			11790787
+17522048:1:NR1H3:FASN	NR1H3	FASN	+		17522048
+17522048:3:NR1H3:FASN	NR1H3	FASN			17522048
+22160584:1:NR1H3:FASN	NR1H3	FASN			22160584
+22160584:5:NR1H3:FASN	NR1H3	FASN	+		22160584
+22160584:8:NR1H3:FASN	NR1H3	FASN	+		22160584
     EOF
 
     TmpFile.with_file(content) do |filename|
-      tsv = TSV.open(filename, :sep => /\s+/)
+      tsv = TSV.open(filename, :key_field => "Transcription Factor (Associated Gene Name)", :fields => ["Target Gene (Associated Gene Name)", "Sign", "PMID"], :merge => true, :type => :double)
+      unzip = tsv.unzip(0,true)
+      assert_equal unzip["NR1H3:FASN"][0].length, unzip["NR1H3:FASN"][1].length
     end
   end
 

data/test/rbbt/tsv/test_parser.rb

@@ -69,5 +69,33 @@ A B C D E
     end
   end
 
+  def test_merge
+    content =<<-EOF
+#: :type=:double
+#PMID:Sentence number:TF:TG	Transcription Factor (Associated Gene Name)	Target Gene (Associated Gene Name)	Sign	Negation	PMID
+24265317:3:NR1H3:FASN	NR1H3	FASN			24265317
+17522048:0:NR1H3:FASN	NR1H3	FASN	+		17522048
+19903962:0:NR1H3:FASN	NR1H3	FASN			19903962
+19903962:7:NR1H3:FASN	NR1H3	FASN			19903962
+22183856:4:NR1H3:FASN	NR1H3	FASN			22183856
+22641099:4:NR1H3:FASN	NR1H3	FASN	+		22641099
+23499676:8:NR1H3:FASN	NR1H3	FASN	+		23499676
+11790787:5:NR1H3:FASN	NR1H3	FASN			11790787
+11790787:7:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:9:NR1H3:FASN	NR1H3	FASN	+		11790787
+11790787:11:NR1H3:FASN	NR1H3	FASN			11790787
+17522048:1:NR1H3:FASN	NR1H3	FASN	+		17522048
+17522048:3:NR1H3:FASN	NR1H3	FASN			17522048
+22160584:1:NR1H3:FASN	NR1H3	FASN			22160584
+22160584:5:NR1H3:FASN	NR1H3	FASN	+		22160584
+22160584:8:NR1H3:FASN	NR1H3	FASN	+		22160584
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Transcription Factor (Associated Gene Name)", :fields => ["Target Gene (Associated Gene Name)", "Sign", "PMID"], :merge => true, :type => :double)
+      assert_equal 16, tsv["NR1H3"]["Sign"].length
+    end
+  end
+
 end
 

data/test/rbbt/util/misc/test_indiferent_hash.rb

@@ -0,0 +1,14 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestIndiferentHash < Test::Unit::TestCase
+  def test_recursive
+    a = {:a => {:b => 1 } }
+    IndiferentHash.setup(a)
+
+    assert IndiferentHash === a["a"]
+    assert_equal 1, a[:a]["b"]
+    assert_equal 1, a["a"][:b]
+  end
+end
+

data/test/rbbt/util/test_cmd.rb

@@ -39,7 +39,7 @@ class TestCmd < Test::Unit::TestCase
     assert_nothing_raised ProcessFailed do CMD.cmd('ls -fake_option', :no_fail => true, :pipe => true) end
  
     assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true, :pipe => true).join end
-    assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
+    assert_raise ConcurrentStreamProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
   end
 
   def test_pipes
@@ -78,4 +78,10 @@ line33
   def test_bash
     puts CMD.bash("awk 'test'")
   end
+
+  def test_cmd_error
+    assert_raise ConcurrentStreamProcessFailed do
+      CMD.cmd_log("ruby -e 'puts 1; STDERR.puts 1; sleep 2; raise'")
+    end
+  end
 end

data/test/rbbt/util/test_open.rb

@@ -98,7 +98,7 @@ class TestOpen < Test::Unit::TestCase
 4
     EOF
     TmpFile.with_file(content) do |file|
-      `bgzip #{file}`
+      `gzip #{file}`
       assert_equal(content, Open.read(file + '.gz'))
       puts content
       FileUtils.rm file + '.gz'
@@ -181,8 +181,8 @@ class TestOpen < Test::Unit::TestCase
       Misc.consume_stream(StringIO.new(text), false, file)
 
       assert_equal text, Open.read(file)
-      assert !File.exist?(file)
-      assert Open.exists? file
+      assert Open.exists?(file)
+      refute File.exist?(file)
     end
 
   end

data/test/rbbt/util/test_ssh.rb

@@ -0,0 +1,10 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSSH < Test::Unit::TestCase
+  def test_marshal
+
+    assert TrueClass === SSHLine.rbbt('turbo', 'true')
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.37.16
+  version: 5.38.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-30 00:00:00.000000000 Z
+date: 2023-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -530,6 +530,7 @@ files:
 - test/rbbt/util/misc/test_communication.rb
 - test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_format.rb
+- test/rbbt/util/misc/test_indiferent_hash.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_omics.rb
@@ -556,6 +557,7 @@ files:
 - test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/test_ssh.rb
 - test/rbbt/util/test_tmpfile.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/step/test_save_load_inputs.rb
@@ -587,7 +589,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.8
+rubygems_version: 3.4.13
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
@@ -608,6 +610,7 @@ test_files:
 - test/rbbt/util/test_procpath.rb
 - test/rbbt/util/python/test_util.rb
 - test/rbbt/util/misc/test_development.rb
+- test/rbbt/util/misc/test_indiferent_hash.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_serialize.rb
@@ -630,6 +633,7 @@ test_files:
 - test/rbbt/util/test_excel2tsv.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/test_ssh.rb
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/simpleopt/test_parse.rb
 - test/rbbt/util/simpleopt/test_setup.rb