rbbt-util 5.28.5 → 5.28.10

data/lib/rbbt/workflow/util/trace.rb

@@ -0,0 +1,182 @@
+require 'rbbt/util/R'
+
+module Workflow
+  def self.trace(seed_jobs, options = {})
+
+    jobs = []
+    seed_jobs.each{|j| jobs << j; jobs += j.rec_dependencies}
+
+    data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
+    min_start = nil
+    max_done = nil
+    jobs.each do |job|
+      next unless job.info[:done]
+      started = job.info[:started]
+      ddone = job.info[:done]
+
+      code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * "."
+      code = code + '.' + job.name
+
+      data[code] = [job.workflow.to_s, job.task_name, started, ddone]
+      if min_start.nil?
+        min_start = started
+      else
+        min_start = started if started < min_start
+      end
+
+      if max_done.nil?
+        max_done = ddone
+      else
+        max_done = ddone if ddone > max_done
+      end
+    end
+
+    data.add_field "Start.second" do |k,value|
+      value["Start"] - min_start
+    end
+
+    data.add_field "End.second" do |k,value|
+      value["End"] - min_start
+    end
+
+    if options[:fix_gap]
+      ranges = []
+      data.through do |k,values|
+        start, eend = values.values_at "Start.second", "End.second"
+
+        ranges << (start..eend)
+      end
+
+      gaps = {}
+      last = nil
+      Misc.collapse_ranges(ranges).each do |range|
+        start = range.begin
+        eend = range.end
+        if last
+          gaps[last] = start - last
+        end
+        last = eend
+      end
+
+      data.process "End.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+
+      data.process "Start.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+    end
+
+    tasks_info = {}
+
+    jobs.each do |dep|
+      next unless dep.info[:done]
+      task = [dep.workflow, dep.task_name].compact.collect{|s| s.to_s} * "#"
+      info = tasks_info[task] ||= {}
+
+      time = dep.info[:done] - dep.info[:started]
+      info[:time] ||= []
+      info[:time] << time
+
+      cpus = nil
+      spark = false
+      shard = false
+      dep.info[:config_keys].select do |kinfo| 
+        key, value, tokens = kinfo
+        key = key.to_s
+        cpus = value if key.include? 'cpu'
+        spark = value if key == 'spark'
+        shard = value if key == 'shard'
+      end
+
+      info[:cpus] = cpus || 1
+      info[:spark] = spark
+      info[:shard] = shard
+    end
+
+    stats = TSV.setup({}, "Task~Calls,Avg. Time,Total Time,Cpus,Spark,Shard#:type=:list")
+
+    tasks_info.each do |task, info|
+      time_lists, cpus, spark, shard = info.values_at :time, :cpus, :spark, :shard
+      avg_time = Misc.mean(time_lists)
+      total_time = Misc.sum(time_lists)
+      calls = time_lists.length
+      stats[task] = [calls, avg_time, total_time, cpus, spark, shard]
+    end
+
+    raise "No jobs to process" if data.size == 0
+
+    start = data.column("Start.second").values.flatten.collect{|v| v.to_f}.min
+    eend = data.column("End.second").values.flatten.collect{|v| v.to_f}.max
+    total = eend - start
+    Log.info "Total time elapsed: #{total} seconds"
+
+    if options[:fix_gap]
+      total_gaps = Misc.sum(gaps.collect{|k,v| v})
+      Log.info "Total gaps: #{total_gaps} seconds"
+    end
+
+    plot, width, height = options.values_at :plot, :width, :height
+    if plot
+      data.R <<-EOF, [:svg]
+    rbbt.require('tidyverse')
+    rbbt.require('ggplot2')
+
+    names(data) <- make.names(names(data))
+    data$id = rownames(data)
+    data$content = data$Task
+    data$start = data$Start
+    data$end = data$End
+    data$Project = data$Workflow
+
+    tasks = data
+
+    #theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
+    theme_gantt <- function(base_size=11, base_family="Sans Serif") {
+      ret <- theme_bw(base_size, base_family) %+replace%
+        theme(panel.background = element_rect(fill="#ffffff", colour=NA),
+              axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
+              title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
+              panel.border = element_blank(), axis.line=element_blank(),
+              panel.grid.minor=element_blank(),
+              panel.grid.major.y = element_blank(),
+              panel.grid.major.x = element_line(size=0.5, colour="grey80"),
+              axis.ticks=element_blank(),
+              legend.position="bottom", 
+              axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
+              strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
+              strip.background=element_rect(fill="#ffffff", colour=NA),
+              panel.spacing.y=unit(1.5, "lines"),
+              legend.key = element_blank())
+
+      ret
+    }
+
+    tasks.long <- tasks %>%
+    gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
+    arrange(date.type, task.date) %>%
+    mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
+
+    x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
+
+    timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
+      geom_segment() + 
+      geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
+      guides(colour=guide_legend(title=NULL)) +
+      labs(x=NULL, y=NULL) + 
+      theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
+
+    rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, pointsize=6)
+      EOF
+    end
+
+    if options[:plot_data]
+      data
+    else
+      stats
+    end
+
+  end
+end

data/share/rbbt_commands/ansible

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Run ansible
+
+$ #{$0} [options] <rules.yml|->
+
+Use - to read from STDIN
+
+-h--help Print this help
+-dr--dry_run Only print the playbook
+-H--hosts* hosts to run it on
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+require 'rbbt/workflow/integration/ansible'
+
+playbook, task = ARGV
+playbook_file = Rbbt.share.ansible["test.yaml"].find if playbook.nil?
+
+dry_run = options.delete :dry_run
+
+if ! Misc.filename?(playbook_file)
+  playbook_file = Rbbt.share.ansible.glob(playbook + ".*").first
+end
+
+if ! Misc.filename?(playbook_file)
+  recipe = Rbbt.share.recipes.glob(playbook + ".*").first
+  playbook_file = Ansible.playbook recipe, task, options
+end
+
+
+if dry_run
+  if Hash === playbook_file
+    ppp playbook_file.to_yaml
+  else
+    ppp Log.color :magenta, playbook_file
+    ppp Open.read(playbook_file) 
+  end
+else
+  Ansible.play playbook_file
+end

data/share/rbbt_commands/purge_job

@@ -3,7 +3,6 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
-require 'rbbt/workflow/remote/ssh/get'
 
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -14,9 +13,7 @@ Remove a job and its dependencies
 $ rbbt purge [options] <job_path> 
 
 -h--help Print this help
--t--test Do a verbose dry run
--r--relocate Include relocated dependencies
-
+-r--recursive Remove recursively
 EOF
 if options[:help]
   if defined? rbbt_usage
@@ -31,4 +28,4 @@ path, search_path, _sep, *other = ARGV
 
 raise ParameterException, "No path given" if path.nil? 
 
-Step.purge(path, options[:relocate])
+Step.purge(path, options[:recursive])

data/share/rbbt_commands/system/status

@@ -201,7 +201,7 @@ workflows.sort.each do |workflow,tasks|
                    Step::INFO_SERIALIZER.load(f)
                  end
                rescue
-                 Log.exception $!
+                 #Log.exception $!
                  {:status => :noinfo}
                end
         IndiferentHash.setup(info)

data/share/rbbt_commands/workflow/forget_deps

@@ -13,6 +13,8 @@ Make a job forget all its dependencies and archive their meta-data
 $ #{$0} [options] <job_path>
 
 -h--help Print this help
+-p--purge Purge dependencies
+-r--recursive_purge Purge dependencies recursively
 
 EOF
 if options[:help]
@@ -29,8 +31,13 @@ path = ARGV[0]
 raise ParameterException, "No path given" if path.nil?
 step = Workflow.load_step path
 
-
-iif step.rec_dependencies
-
 step.archive_deps
+step.copy_files_dir
+dependencies = step.dependencies
 step.set_info :dependencies, []
+
+if options[:purge]
+  dependencies.each do |dependency|
+    Step.purge(dependency.path, options[:recursive_purge])
+  end
+end

data/share/rbbt_commands/workflow/server

@@ -25,6 +25,7 @@ $ rbbt workflow server [options] <Workflow>
 --stream Activate streaming of workflow tasks
 -fs--file_server Activate file serving for resources
 -mj--monitor_jobs Monitor jobs (UNSAFE)
+-a--app_dir* Application execution directory
 --export_all Export all workflow tasks (use with caution!)
 --export* Export workflow tasks (asynchronous)
 --export_asynchronous* Export workflow tasks as asynchronous
@@ -74,6 +75,7 @@ sync_exports = options[:export_synchronous].split(/\s*,/) if options[:export_syn
 exec_exports = options[:export_exec].split(/\s*,/) if options[:export_exec]
 
 TmpFile.with_file do |app_dir|
+  app_dir = options[:app_dir] if options[:app_dir]
   Misc.in_dir(app_dir) do
     app_dir = Path.setup(app_dir.dup)
     Open.write(app_dir.etc.target_workflow.find, workflow)

data/test/rbbt/association/test_index.rb

@@ -73,21 +73,21 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_index_flat
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(file,  :type => :flat, :source => "Transcription Factor Associated Gene Name=~Associated Gene Name", :merge => true)
     assert tsv.match("TP53").length > 10
   end
 
   def test_index_flat_to_matrix
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(file,  :type => :flat, :source => "Transcription Factor Associated Gene Name=~Associated Gene Name", :merge => true)
     assert(tsv.to_matrix(false))
   end
 
   def test_filter_no_block
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter :directed?
@@ -96,7 +96,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_filter_no_block_value
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter :Effect, "inhibition"
@@ -105,7 +105,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_filter_block_value_field
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter :Effect do |value|
@@ -116,7 +116,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_filter_block_no_value_field
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     tsv = Association.index(EFFECT, EFFECT_OPTIONS.merge(:undirected => false, :source => "SG=~Associated Gene Name", :target => "TG=~Associated Gene Name"), :persist => true)
     tsv.unnamed = false
     matches = tsv.filter do |key,values|

data/test/rbbt/knowledge_base/test_query.rb

@@ -22,7 +22,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
     :namespace => "Hsa"
   }
 
-  EFFECT_TSV = TSV.open EFFECT, EFFECT_OPTIONS.dup 
+  EFFECT_TSV = TSV.open EFFECT, EFFECT_OPTIONS.dup
 
   KNOWLEDGE_BASE = KnowledgeBase.new Rbbt.tmp.test.kb_foo2, "Hsa"
   KNOWLEDGE_BASE.format = {"Gene" => "Ensembl Gene ID"}
@@ -34,11 +34,11 @@ TP53 NFKB1|GLI1 activation|activation true|true
       assert_equal 6, KNOWLEDGE_BASE.subset(:effects, :all).length
 
       assert_equal 4, KNOWLEDGE_BASE.subset(:effects, :all).target_entity.uniq.length
-      assert_equal %w(Effect), KNOWLEDGE_BASE.subset(:effects, :all).info.first.keys 
+      assert_equal %w(Effect), KNOWLEDGE_BASE.subset(:effects, :all).info.first.keys
     end
   end
 
-  def test_subset_all_persist_format
+  def _test_subset_all_persist_format
     assert KNOWLEDGE_BASE.subset(:effects, :all).target_entity.reject{|e| e =~ /^ENS/}.empty?
   end
 

data/test/rbbt/knowledge_base/test_registry.rb

@@ -45,7 +45,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
 
   def test_index_flat
     require 'rbbt/sources/tfacts'
-    file = TFacts.regulators
+    file = TFactS.regulators
     KNOWLEDGE_BASE.register :tfacts, file,  :type => :flat, :source => "Transcription Factor Associated Gene Name=~Associated Gene Name", :merge => true
     assert KNOWLEDGE_BASE.subset(:tfacts, :source => ["TP53"], :target => :all).length > 10
   end

data/test/rbbt/persist/tsv/test_cdb.rb

@@ -4,19 +4,12 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
-  extend Annotation
-
-  self.annotation :test_annotation
-end
-
 class TestPersistTSVCDB < Test::Unit::TestCase
 
   if Persist.respond_to? :open_cbd
     def test_organism
       require 'rbbt/sources/organism'
       TmpFile.with_file nil do |tmp_file|
-        file = CMD.cmd("head -n 1000000", :in => Organism.identifiers("Hsa").open, :pipe => true)
         tsv = Organism.identifiers("Hsa").tsv(:key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "CDB", :persist_dir => tmp_file)
         assert_equal "ENSG00000141510", tsv["TP53"]
       end

data/test/rbbt/persist/tsv/test_kyotocabinet.rb

@@ -5,15 +5,9 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
-  extend Annotation
-
-  self.annotation :test_annotation
-end
-
 class TestPersistTSVKC < Test::Unit::TestCase
   if Persist.respond_to? :open_kyotocabinet
-    def _test_organism_kch
+    def test_organism_kch
       require 'rbbt/sources/organism'
       TmpFile.with_file do |tmp_file|
         tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kch", :persist_dir => tmp_file
@@ -22,7 +16,7 @@ class TestPersistTSVKC < Test::Unit::TestCase
     end
 
 
-    def _test_organism_kct
+    def test_organism_kct
       require 'rbbt/sources/organism'
       TmpFile.with_file do |tmp_file|
         tsv = Organism.identifiers("Hsa").tsv :key_field => "Associated Gene Name", :fields => ["Ensembl Gene ID"], :type => :single, :persist => true, :persist_engine => "kct", :persist_dir => tmp_file

data/test/rbbt/persist/tsv/test_leveldb.rb

@@ -4,12 +4,6 @@ require 'rbbt/annotations'
 require 'rbbt/util/tmpfile'
 require 'test/unit'
 
-module TestAnnotation
-  extend Annotation
-
-  self.annotation :test_annotation
-end
-
 class TestPersistTSVLevelDB < Test::Unit::TestCase
 
   if Persist.respond_to? :open_leveldb