rbbt-util 5.28.7 → 5.28.12

data/lib/rbbt/workflow/util/provenance.rb

@@ -66,7 +66,7 @@ class Step
     str << "\n"
   end
 
-  def self.prov_report(step, offset = 0, task = nil, seen = [])
+  def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false)
     info = step.info  || {}
     info[:task_name] = task
     path  = step.path
@@ -82,9 +82,13 @@ class Step
       new = ! seen.include?(path)
       if new
         seen << path
-        str << prov_report(dep, offset + 1, task, seen)
+        str << prov_report(dep, offset + 1, task, seen, expand_repeats)
       else
-        str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+        if expand_repeats
+          str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
+        else
+          str << Log.color(:green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+        end
       end
     end if step.dependencies
     str

data/share/rbbt_commands/ansible

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Run ansible
+
+$ #{$0} [options] <rules.yml|->
+
+Use - to read from STDIN
+
+-h--help Print this help
+-dr--dry_run Only print the playbook
+-H--hosts* hosts to run it on
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+require 'rbbt/workflow/integration/ansible'
+
+playbook, task = ARGV
+playbook_file = Rbbt.share.ansible["test.yaml"].find if playbook.nil?
+
+dry_run = options.delete :dry_run
+
+if ! Misc.filename?(playbook_file)
+  playbook_file = Rbbt.share.ansible.glob(playbook + ".*").first
+end
+
+if ! Misc.filename?(playbook_file)
+  recipe = Rbbt.share.recipes.glob(playbook + ".*").first
+  playbook_file = Ansible.playbook recipe, task, options
+end
+
+
+if dry_run
+  if Hash === playbook_file
+    ppp playbook_file.to_yaml
+  else
+    ppp Log.color :magenta, playbook_file
+    ppp Open.read(playbook_file) 
+  end
+else
+  Ansible.play playbook_file
+end

data/share/rbbt_commands/purge_job

@@ -14,7 +14,6 @@ $ rbbt purge [options] <job_path>
 
 -h--help Print this help
 -r--recursive Remove recursively
-
 EOF
 if options[:help]
   if defined? rbbt_usage

data/share/rbbt_commands/system/status

@@ -49,27 +49,27 @@ def pid_msg(pid)
 end
 
 
-def status_msg(status)
-  color = case status.to_sym
-          when :error, :aborted, :missing, :dead, :broken
-            :red
-          when :streaming, :started
-            :cyan
-          when :done
-            :green
-          when :noinfo
-            :blue
-          when :dependencies, :waiting, :setup
-            :yellow
-          else
-            if status.to_s.index ">"
-              :cyan
-            else
-              :cyan
-            end
-          end
-  Log.color(color, status.to_s)
-end
+#def status_msg(status)
+#  color = case status.to_sym
+#          when :error, :aborted, :missing, :dead, :broken
+#            :red
+#          when :streaming, :started
+#            :cyan
+#          when :done
+#            :green
+#          when :noinfo, :notfound
+#            :blue
+#          when :dependencies, :waiting, :setup
+#            :yellow
+#          else
+#            if status.to_s.index ">"
+#              :cyan
+#            else
+#              :cyan
+#            end
+#          end
+#  Log.color(color, status.to_s)
+#end
 
 def input_msg(file, inputs)
 
@@ -218,7 +218,7 @@ workflows.sort.each do |workflow,tasks|
             status << Log.color(:red, " (dead)") 
           end
         end
-        str << " #{ status_msg status }"
+        str << " #{ Step.prov_status_msg status }"
         str << " (dirty)" if status == 'done' && Workflow.load_step(file).dirty?
 
         if inputs and inputs.any? 

data/share/rbbt_commands/workflow/forget_deps

@@ -13,6 +13,8 @@ Make a job forget all its dependencies and archive their meta-data
 $ #{$0} [options] <job_path>
 
 -h--help Print this help
+-p--purge Purge dependencies
+-r--recursive_purge Purge dependencies recursively
 
 EOF
 if options[:help]
@@ -29,8 +31,13 @@ path = ARGV[0]
 raise ParameterException, "No path given" if path.nil?
 step = Workflow.load_step path
 
-
-iif step.rec_dependencies
-
 step.archive_deps
+step.copy_files_dir
+dependencies = step.dependencies
 step.set_info :dependencies, []
+
+if options[:purge]
+  dependencies.each do |dependency|
+    Step.purge(dependency.path, options[:recursive_purge])
+  end
+end

data/share/rbbt_commands/workflow/info

@@ -17,6 +17,7 @@ $ rbbt workflow info <job-result>
 -a--all Print all info entries
 -r--recursive Print recursive input values
 -o--original Print original object
+-w--width* Screen width
 EOF
 
 SOPT.usage if options[:help]
@@ -24,6 +25,7 @@ SOPT.usage if options[:help]
 file = ARGV.shift
 all = options.delete :all
 recursive = options.delete :recursive
+width = (options.delete(:width) || 80).to_i
 
 def get_step(file)
   file = file.sub(/\.(info|files)/,'')
@@ -86,6 +88,7 @@ pid = info[:pid]
 exception = info[:exception]
 rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
 
+
 puts Log.color(:magenta, "File") << ": " << step.path
 puts Log.color(:magenta, "Status") << ": " << status_msg(status) << ((step.aborted? || step.error?) && step.recoverable_error? ? " (recoverable)" : "" ) << (step.dirty? ? " (dirty)" : "")
 puts Log.color(:magenta, "Pid") << ": " << pid_msg(pid, status.to_s == "done")
@@ -96,14 +99,14 @@ if inputs and inputs.any?
   inputs.each do |input,value|
     case value
     when nil
-      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', width, 20, :blue)
     when Array
-      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), width, 20, :blue)
     when TrueClass, FalseClass
-      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, width, 20, :blue)
     else
       text = value.to_s.split("\n")[0..5].compact * "\n\n"
-      puts Misc.format_definition_list_item("  " + input.to_s, text, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, text, width, 20, :blue)
     end
   end
 end
@@ -162,16 +165,16 @@ if recursive
     inputs.each do |input,value|
       case value
       when nil
-        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', width, 20, :blue)
       when Array
-        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), width, 20, :blue).gsub("\n\n","\n")
       when TrueClass, FalseClass
-        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, width, 20, :blue)
       else
-        lines = value.to_s.split("\n").collect{|l| l.length >= 60 ? l[0..45] + " ..." : l }
+        lines = value.to_s.split("\n").collect{|l| l.length >= width - 5 ? l[0..width - 5] + " ..." : l }
         text = lines[0..5].compact * "\n\n"
         text << "\n\n...\n\n" if lines.length > 6
-        puts Misc.format_definition_list_item("  " << input.to_s, text, 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, text, width, 20, :blue).gsub("\n\n","\n")
       end
     end
   end

data/share/rbbt_commands/workflow/prov

@@ -24,6 +24,7 @@ $ rbbt workflow prov <job-result>
 -i--inputs* List of inputs to print
 -if--info_fields* List of info fields to print
 -t--touch Update modification times to be consistent
+-e--expand_repeats Show all the dependency tree even if reapeated dependencies have already been seen before
 EOF
 
 SOPT.usage if options[:help]
@@ -130,6 +131,6 @@ if options[:plot]
   end
 
 else
-  puts Step.prov_report(step).strip
+  puts Step.prov_report(step, 0, nil, [], options[:expand_repeats]).strip
 end
 

data/test/rbbt/association/test_index.rb

@@ -73,21 +73,21 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_index_flat
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(file,  :type => :flat, :source => "Transcription Factor Associated Gene Name=~Associated Gene Name", :merge => true)
     assert tsv.match("TP53").length > 10
   end
 
   def test_index_flat_to_matrix
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(file,  :type => :flat, :source => "Transcription Factor Associated Gene Name=~Associated Gene Name", :merge => true)
     assert(tsv.to_matrix(false))
   end
 
   def test_filter_no_block
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter :directed?
@@ -96,7 +96,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_filter_no_block_value
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter :Effect, "inhibition"
@@ -105,7 +105,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_filter_block_value_field
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter :Effect do |value|
@@ -116,7 +116,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_filter_block_no_value_field
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter do |key,values|

data/test/rbbt/knowledge_base/test_query.rb

@@ -22,7 +22,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
     :namespace => "Hsa"
   }
 
-  EFFECT_TSV = TSV.open EFFECT, EFFECT_OPTIONS.dup 
+  EFFECT_TSV = TSV.open EFFECT, EFFECT_OPTIONS.dup
 
   KNOWLEDGE_BASE = KnowledgeBase.new Rbbt.tmp.test.kb_foo2, "Hsa"
   KNOWLEDGE_BASE.format = {"Gene" => "Ensembl Gene ID"}
@@ -34,11 +34,11 @@ TP53 NFKB1|GLI1 activation|activation true|true
       assert_equal 6, KNOWLEDGE_BASE.subset(:effects, :all).length
 
       assert_equal 4, KNOWLEDGE_BASE.subset(:effects, :all).target_entity.uniq.length
-      assert_equal %w(Effect), KNOWLEDGE_BASE.subset(:effects, :all).info.first.keys 
+      assert_equal %w(Effect), KNOWLEDGE_BASE.subset(:effects, :all).info.first.keys
     end
   end
 
-  def test_subset_all_persist_format
+  def _test_subset_all_persist_format
     assert KNOWLEDGE_BASE.subset(:effects, :all).target_entity.reject{|e| e =~ /^ENS/}.empty?
   end
 

data/test/rbbt/knowledge_base/test_registry.rb

@@ -45,7 +45,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_index_flat
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     KNOWLEDGE_BASE.register :tfacts, file,  :type => :flat, :source => "Transcription Factor Associated Gene Name=~Associated Gene Name", :merge => true
     assert KNOWLEDGE_BASE.subset(:tfacts, :source => ["TP53"], :target => :all).length > 10
   end

data/test/rbbt/persist/tsv/test_cdb.rb

@@ -4,19 +4,12 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
-  extend Annotation
-
-  self.annotation :test_annotation
-end
-
 class TestPersistTSVCDB < Test::Unit::TestCase
 
   if Persist.respond_to? :open_cbd
     def test_organism
       require 'rbbt/sources/organism'
       TmpFile.with_file nil do |tmp_file|
-        file = CMD.cmd("head -n 1000000", :in => Organism.identifiers("Hsa").open, :pipe => true)
         tsv = Organism.identifiers("Hsa").tsv(:key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "CDB", :persist_dir => tmp_file)
         assert_equal "ENSG00000141510", tsv["TP53"]
       end

data/test/rbbt/persist/tsv/test_kyotocabinet.rb

@@ -5,15 +5,9 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
-  extend Annotation
-
-  self.annotation :test_annotation
-end
-
 class TestPersistTSVKC < Test::Unit::TestCase
   if Persist.respond_to? :open_kyotocabinet
-    def _test_organism_kch
+    def test_organism_kch
       require 'rbbt/sources/organism'
       TmpFile.with_file do |tmp_file|
         tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kch", :persist_dir => tmp_file
@@ -22,7 +16,7 @@ class TestPersistTSVKC < Test::Unit::TestCase
     end
 
 
-    def _test_organism_kct
+    def test_organism_kct
       require 'rbbt/sources/organism'
       TmpFile.with_file do |tmp_file|
         tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kct", :persist_dir => tmp_file

data/test/rbbt/persist/tsv/test_leveldb.rb

@@ -4,12 +4,6 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
-  extend Annotation
-
-  self.annotation :test_annotation
-end
-
 class TestPersistTSVLevelDB < Test::Unit::TestCase
 
   if Persist.respond_to? :open_leveldb

data/test/rbbt/persist/tsv/test_lmdb.rb

@@ -4,12 +4,6 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
-  extend Annotation
-
-  self.annotation :test_annotation
-end
-
 class TestPersistTSVLMDB < Test::Unit::TestCase
 
   def test_organism

data/test/rbbt/persist/tsv/test_tokyocabinet.rb

@@ -4,7 +4,8 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
+
+module TestAnnotationModule
   extend Annotation
 
   self.annotation :test_annotation
@@ -25,8 +26,8 @@ class TestPersistTSVTC < Test::Unit::TestCase
       entity1 = "Entity 1"
       entity2 = "Entity 2"
 
-      TestAnnotation.setup(entity1, :test_annotation => "1")
-      TestAnnotation.setup(entity2, :test_annotation => "2")
+      TestAnnotationModule.setup(entity1, :test_annotation => "1")
+      TestAnnotationModule.setup(entity2, :test_annotation => "2")
 
       annotations = [entity1, entity2]
 
@@ -56,9 +57,9 @@ class TestPersistTSVTC < Test::Unit::TestCase
       entity2 = "Entity 2"
       entity2bis = "Entity 2"
 
-      TestAnnotation.setup(entity1, :test_annotation => "1")
-      TestAnnotation.setup(entity2, :test_annotation => "2")
-      TestAnnotation.setup(entity2bis, :test_annotation => "2")
+      TestAnnotationModule.setup(entity1, :test_annotation => "1")
+      TestAnnotationModule.setup(entity2, :test_annotation => "2")
+      TestAnnotationModule.setup(entity2bis, :test_annotation => "2")
 
       annotations = [entity1, entity2, entity2bis]
 
@@ -105,8 +106,8 @@ class TestPersistTSVTC < Test::Unit::TestCase
       entity1 = "Entity 1"
       entity2 = "Entity 2"
 
-      TestAnnotation.setup(entity1, :test_annotation => "1")
-      TestAnnotation.setup(entity2, :test_annotation => "2")
+      TestAnnotationModule.setup(entity1, :test_annotation => "1")
+      TestAnnotationModule.setup(entity2, :test_annotation => "2")
 
       annotations = [entity1, entity2]
 
@@ -138,7 +139,7 @@ class TestPersistTSVTC < Test::Unit::TestCase
       entity2 = "Entity 2"
 
       annotations = [entity1, entity2]
-      TestAnnotation.setup(annotations, :test_annotation => "1")
+      TestAnnotationModule.setup(annotations, :test_annotation => "1")
       annotations.extend AnnotatedArray
 
       persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
@@ -171,11 +172,11 @@ class TestPersistTSVTC < Test::Unit::TestCase
       entity2 = "Entity 2"
 
       annotations = [entity1, entity2]
-      TestAnnotation.setup(annotations, :test_annotation => "1")
+      TestAnnotationModule.setup(annotations, :test_annotation => "1")
       annotations.extend AnnotatedArray
 
       annotations_ary = [annotations]
-      TestAnnotation.setup(annotations_ary, :test_annotation => "1")
+      TestAnnotationModule.setup(annotations_ary, :test_annotation => "1")
       annotations_ary.extend AnnotatedArray
 
       persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
@@ -210,9 +211,9 @@ class TestPersistTSVTC < Test::Unit::TestCase
       entity2 = "Entity 2"
       entity2bis = "Entity 2"
 
-      TestAnnotation.setup(entity1, :test_annotation => "1")
-      TestAnnotation.setup(entity2, :test_annotation => "2")
-      TestAnnotation.setup(entity2bis, :test_annotation => "2")
+      TestAnnotationModule.setup(entity1, :test_annotation => "1")
+      TestAnnotationModule.setup(entity2, :test_annotation => "2")
+      TestAnnotationModule.setup(entity2bis, :test_annotation => "2")
 
       annotations = [entity1, entity2, entity2bis]