rbbt-util 5.28.10 → 5.29.1

data/share/rbbt_commands/workflow/info

@@ -17,6 +17,7 @@ $ rbbt workflow info <job-result>
 -a--all Print all info entries
 -r--recursive Print recursive input values
 -o--original Print original object
+-w--width* Screen width
 EOF
 
 SOPT.usage if options[:help]
@@ -24,6 +25,7 @@ SOPT.usage if options[:help]
 file = ARGV.shift
 all = options.delete :all
 recursive = options.delete :recursive
+width = (options.delete(:width) || 80).to_i
 
 def get_step(file)
   file = file.sub(/\.(info|files)/,'')
@@ -86,6 +88,7 @@ pid = info[:pid]
 exception = info[:exception]
 rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
 
+
 puts Log.color(:magenta, "File") << ": " << step.path
 puts Log.color(:magenta, "Status") << ": " << status_msg(status) << ((step.aborted? || step.error?) && step.recoverable_error? ? " (recoverable)" : "" ) << (step.dirty? ? " (dirty)" : "")
 puts Log.color(:magenta, "Pid") << ": " << pid_msg(pid, status.to_s == "done")
@@ -96,14 +99,14 @@ if inputs and inputs.any?
   inputs.each do |input,value|
     case value
     when nil
-      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', width, 20, :blue)
     when Array
-      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), width, 20, :blue)
     when TrueClass, FalseClass
-      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, width, 20, :blue)
     else
       text = value.to_s.split("\n")[0..5].compact * "\n\n"
-      puts Misc.format_definition_list_item("  " + input.to_s, text, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, text, width, 20, :blue)
     end
   end
 end
@@ -162,16 +165,16 @@ if recursive
     inputs.each do |input,value|
       case value
       when nil
-        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', width, 20, :blue)
       when Array
-        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), width, 20, :blue).gsub("\n\n","\n")
       when TrueClass, FalseClass
-        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, width, 20, :blue)
       else
-        lines = value.to_s.split("\n").collect{|l| l.length >= 60 ? l[0..45] + " ..." : l }
+        lines = value.to_s.split("\n").collect{|l| l.length >= width - 5 ? l[0..width - 5] + " ..." : l }
         text = lines[0..5].compact * "\n\n"
         text << "\n\n...\n\n" if lines.length > 6
-        puts Misc.format_definition_list_item("  " << input.to_s, text, 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, text, width, 20, :blue).gsub("\n\n","\n")
       end
     end
   end

data/share/rbbt_commands/workflow/prov

@@ -24,6 +24,7 @@ $ rbbt workflow prov <job-result>
 -i--inputs* List of inputs to print
 -if--info_fields* List of info fields to print
 -t--touch Update modification times to be consistent
+-e--expand_repeats Show all the dependency tree even if reapeated dependencies have already been seen before
 EOF
 
 SOPT.usage if options[:help]
@@ -130,6 +131,6 @@ if options[:plot]
   end
 
 else
-  puts Step.prov_report(step).strip
+  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats]).strip
 end
 

data/test/rbbt/test_workflow.rb

@@ -156,6 +156,15 @@ for this dependency
     Open.read(file).reverse
   end
 
+  task :create_file => :text do |file|
+    Open.write(file('a'), "A")
+    Open.write(file('b'), "B")
+    "DONE"
+  end
+
+  dep_task :reverse_step_file, TestWF, :reverse_file do |jobname, options, dependencies|
+    dep = dependencies.flatten.first
+  end
 
 end
 
@@ -377,9 +386,9 @@ class TestWorkflow < Test::Unit::TestCase
     job.run
     Misc.with_env "RBBT_UPDATE", 'true' do
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
-      job = TestWF.job(:t3)
-      job.step(:t1).clean
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
       job = TestWF.job(:t3).recursive_clean
       job.run
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
@@ -418,7 +427,7 @@ class TestWorkflow < Test::Unit::TestCase
     TmpFile.with_file do |dir|
       Path.setup(dir)
       Step.save_job_inputs(job, dir)
-      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.read']
+      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.yaml']
       inputs  = Workflow.load_inputs(dir, [:file], :file => :file)
       assert_equal inputs, {:file => 'code'}
     end
@@ -440,4 +449,27 @@ class TestWorkflow < Test::Unit::TestCase
       end
     end
   end
+
+  def test_input_step_file_check
+    job = TestWF.job(:t3).recursive_clean
+    job.run
+    Misc.with_env "RBBT_UPDATE", 'true' do
+      assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      job = TestWF.job(:t3).recursive_clean
+      job.run
+      assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+      job = TestWF.job(:t3)
+      sleep 1
+      Open.touch job.step(:t1).path
+      Misc.with_env "RBBT_UPDATE", "false" do
+        assert job.updated?
+      end
+      Misc.with_env "RBBT_UPDATE", "true" do
+        assert ! job.updated?
+      end
+    end
+  end
 end

data/test/rbbt/tsv/test_attach.rb

@@ -566,7 +566,7 @@ row2,CC
       tsv3.keys.each{|k| tsv3[k] = nil if tsv3[k] == ""}
     end
 
-    assert_equal tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"], [nil, "B", nil]
+    assert_equal [nil, "B", nil], tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"]
   end
 
   def test_attach_index_both_non_key
@@ -597,10 +597,10 @@ A    Id3
 
     tsv1 = tsv2 = nil
 
-    tsv1 = Rbbt.tmp.test.test1.data.tsv :double,  :sep => /\s+/
-    tsv2 = Rbbt.tmp.test.test2.data.tsv :double,  :sep => /\s+/
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
 
-    tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce.find #.to_s
+    tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce(true).produce.find #.to_s
 
     tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
     Log.tsv tsv1
@@ -627,12 +627,92 @@ E    B
 
     tsv1 = tsv2 = nil
 
-    tsv1 = Rbbt.tmp.test.test1.data.tsv :double,  :sep => /\s+/
-    tsv2 = Rbbt.tmp.test.test2.data.tsv :double,  :sep => /\s+/
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
 
     tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
     Log.tsv tsv1
     
   end
+
+  def test_attach_complete
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA
+row1    a|aa|aaa
+row2    A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueB
+row1    b
+row3    C
+    EOF
+    Rbbt.claim Rbbt.tmp.test.test1.data, :string, content1
+    Rbbt.claim Rbbt.tmp.test.test2.data, :string, content2
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.attach tsv2, :complete => true
+    assert_equal [[], ["C"]], tsv1["row3"]
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+
+    ppp tsv1.attach tsv2, :complete => ["AA"]
+    tsv1.attach tsv2, :complete => ["AA"]
+    assert_equal [["AA"], ["C"]], tsv1["row3"]
+  end
+
+  def test_attach_complete_identifiers
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA
+row1    a|aa|aaa
+row2    A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#Id2    ValueB
+ROW_1    b
+ROW_2    C
+    EOF
+
+    identifiers =<<-EOF
+#: :sep=/\\s+/
+#Id    Id2
+row1    ROW_1
+row2    ROW_2
+row3    ROW_3
+    EOF
+    Rbbt.claim Rbbt.tmp.test.test1.data, :string, content1
+    Rbbt.claim Rbbt.tmp.test.test2.data, :string, content2
+    Rbbt.claim Rbbt.tmp.test.identifiers.data, :string, identifiers
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+    ids = Rbbt.tmp.test.identifiers.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1.attach tsv2
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+    ids = Rbbt.tmp.test.identifiers.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1.attach tsv2, :complete => true
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.10
+  version: 5.29.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-17 00:00:00.000000000 Z
+date: 2021-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -211,6 +211,8 @@ files:
 - lib/rbbt/entity/identifiers.rb
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/hpc.rb
+- lib/rbbt/hpc/orchestrate.rb
+- lib/rbbt/hpc/slurm.rb
 - lib/rbbt/knowledge_base.rb
 - lib/rbbt/knowledge_base/enrichment.rb
 - lib/rbbt/knowledge_base/entity.rb
@@ -376,6 +378,9 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
+- share/rbbt_commands/slurm/list
+- share/rbbt_commands/slurm/orchestrate
+- share/rbbt_commands/slurm/task
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/boxplot
 - share/rbbt_commands/stat/compare_lists
@@ -430,7 +435,6 @@ files:
 - share/rbbt_commands/workflow/remote/list
 - share/rbbt_commands/workflow/remote/remove
 - share/rbbt_commands/workflow/server
-- share/rbbt_commands/workflow/slurm
 - share/rbbt_commands/workflow/task
 - share/rbbt_commands/workflow/trace
 - share/unicorn.rb