rbbt-util 5.25.45 → 5.25.46

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1fec18b4fe1096cc53f3f100a2caabdf0fa52c17
-  data.tar.gz: e484faf85e964516add9d7c9be0f0a7e37b172f8
+  metadata.gz: d7a6bfb0c182d36d1323ddefe70c93f57e173c55
+  data.tar.gz: fff96627b902d989f53d92a4fe6f327502f75c89
 SHA512:
-  metadata.gz: 7b199196a490f4a6a3ffcc4fae923afc7802548bbf3463f64b1e76dc03f6e53ec4052b527acce38d10d62c55d5b2d19f89e6e738ff0a615007d48f1dda133aab
-  data.tar.gz: 072d28fb2a80c8b39bb26db1f5c607daa4c136f5763bbf6e79c19f2cbf4e486b9bc9ceaf4ac6e7b5aebf4aa6db9ee7fdfd40bfd8dd22f45d0afdab1782d9b864
+  metadata.gz: 9cd88404da356142456ea01f3efdb78ec784613a4b7670a4826ca40a4a8d74275632c7cf7ba521437b1554332e713b02904da2735fc30489bdaed9cc858f5e9e
+  data.tar.gz: 26d5b2fc59d6cd3446dfb7de5094c5dec73b44ab9e438e451f160bee3b9a3eddbdb3b49f5614bfc4fb40048bafe7b745360185a203147c0a9d9f2b23b441ae15

data/lib/rbbt/tsv/excel.rb

@@ -94,10 +94,12 @@ module TSV
 
   module XLS
     def self.read(file, options = {})
-      options = Misc.add_defaults options, :sep2 => /[,|]\s?/
+      options = Misc.add_defaults options, :sep2 => /[,|]\s?/, :skip_rows => 0
       sheet = Misc.process_options options, :sheet
       header = Misc.process_options options, :header
       text = Misc.process_options options, :text
+      skip_rows = Misc.process_options options, :skip_rows
+      skip_rows = skip_rows.to_i
 
       header = true unless header == false
       sheet ||= "0"
@@ -117,6 +119,11 @@ module TSV
         rows = []
 
         sheet.each do |row|
+          if skip_rows > 0
+            skip_rows -= 1
+            next
+          end
+          
           rows << row.values_at(0..(row.size - 1)).collect{|c| String === c ? c.gsub("\n", ' ') : c }
         end
 
@@ -159,10 +166,12 @@ module TSV
 
   module XLSX
     def self.read(file, options = {})
-      options = Misc.add_defaults options, :sep2 => /[,|]\s?/
+      options = Misc.add_defaults options, :sep2 => /[,|]\s?/, :skip_rows => 0
       sheet = Misc.process_options options, :sheet
       header = Misc.process_options options, :header
       text = Misc.process_options options, :text
+      skip_rows = Misc.process_options options, :skip_rows
+      skip_rows = skip_rows.to_i
 
       header = true unless header == false
 
@@ -182,6 +191,11 @@ module TSV
 
         sheet.each do |row|
           next if row.nil?
+          if skip_rows > 0
+            skip_rows -= 1
+            next
+          end
+          
           rows << row.cells.collect{|c| c.nil? ? nil : c.value}.collect{|c| String === c ? c.gsub("\n", ' ') : c }
         end
 

data/lib/rbbt/util/cmd.rb

@@ -139,7 +139,7 @@ module CMD
     pid = io.pids.first
 
     while c = io.getc
-      STDERR << c
+      STDERR << c if Log.severity == 0
       if c == "\n"
         if pid
           Log.logn "STDOUT [#{pid}]: ", 0

data/lib/rbbt/util/log/progress/util.rb

@@ -74,6 +74,10 @@ module Log
       Log::LAST.replace "remove_bar" if Log::LAST == "progress"
     end
 
+    def remove(error = false)
+      Log::ProgressBar.remove_bar self, error
+    end
+
     def self.with_bar(max, options = {})
       bar = new_bar(max, options)
       begin

data/lib/rbbt/util/open.rb

@@ -257,7 +257,7 @@ module Open
     if (dir_sub_path = find_repo_dir(file))
       remove_from_repo(*dir_sub_path)
     else
-      FileUtils.rm(file) if File.exists?(file)
+      FileUtils.rm(file) if File.exists?(file) or Open.broken_link?(file)
     end
   end
 
@@ -793,4 +793,8 @@ module Open
       FileUtils.touch(file)
     end
   end
+
+  def self.broken_link?(path)
+    File.symlink?(path) && ! File.exists?(File.readlink(path))
+  end
 end

data/lib/rbbt/workflow/step.rb

@@ -303,14 +303,15 @@ class Step
       return 
     end
 
-    if Open.exists?(path) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir)
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir)
 
       @result = nil
       @pid = nil
 
       Misc.insist do
         Open.rm info_file if Open.exists? info_file
-        Open.rm path if Open.exists? path
+        Open.rm path if (Open.exists?(path) or Open.broken_link?(path))
+        Open.rm path if (Open.exists?(path) or Open.broken_link?(path))
         Open.rm_rf files_dir if Open.exists? files_dir
         Open.rm pid_file if Open.exists? pid_file
       end

data/share/rbbt_commands/system/clean

@@ -82,7 +82,7 @@ puts
 jobs = Rbbt.job_info workflow, task
 workflows = {}
 
-TSV.traverse jobs, :_bar => "Checking job status" do |file,i|
+TSV.traverse jobs do |file,i|
   if options[:quick] and i[:done]
     status = 'done'
   else
@@ -94,7 +94,7 @@ TSV.traverse jobs, :_bar => "Checking job status" do |file,i|
              {:status => :noinfo}
            end
 
-    done = Open.exists?(file)
+    done = Open.exists?(file) && ! Open.broken_link?(file)
     pid = info[:pid] || (Open.exists?(file + '.pid') && Open.read(file + '.pid')) unless done
 
     status = info[:status].to_s

data/share/rbbt_commands/system/status

@@ -48,7 +48,7 @@ end
 
 def status_msg(status)
   color = case status.to_sym
-          when :error, :aborted, :missing, :dead
+          when :error, :aborted, :missing, :dead, :broken
             :red
           when :streaming, :started
             :cyan
@@ -56,7 +56,7 @@ def status_msg(status)
             :green
           when :noinfo
             :blue
-          when :dependencies, :waiting, :setyp
+          when :dependencies, :waiting, :setup
             :yellow
           else
             if status.to_s.index ">"
@@ -199,7 +199,8 @@ workflows.sort.each do |workflow,tasks|
 
         pid = info[:pid]
         status = info[:status]
-        status = :missing if status == :done and not Open.exist? file
+        status = :missing if status == :done and not (Open.exist?(file) && ! Open.broken_link?(file))
+        status = :broken if Open.broken_link?(file)
         status = status.to_s
         if status != "done" and pid and not Misc.pid_exists?(pid)
           if File.exist? file

data/share/rbbt_commands/tsv/read_excel

@@ -21,6 +21,7 @@ Use - to read from STDIN
 -s2--sep2* Second level separator. Default /[,|]\\s?/
 -h--help Print this help
 -s--sheet* Sheet to extract
+-skip--skip_rows* Initial rows to skip
 
 EOF
 if options[:help]

data/share/rbbt_commands/workflow/trace

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/workflow'
+
+require 'rbbt-util'
+require 'fileutils'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow/step'
+require 'rbbt/workflow/provenance'
+require 'rbbt/util/misc'
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/util/R'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Examine the provenance of a job result
+
+$ rbbt workflow trace <job-result> <file.png>
+
+-h--help Help
+-i--inputs* List of inputs to print
+-if--info_fields* List of info fields to print
+EOF
+
+SOPT.usage if options[:help]
+
+$inputs = (options[:inputs] || "").split(",")
+$info_fields = (options[:info_fields] || "").split(",")
+
+file = ARGV.shift || "/home/mvazque2/.rbbt/var/jobs/Sample/haplotype/NA12878"
+plot = ARGV.shift || "/tmp/plot.png"
+
+$seen = []
+def get_step(file)
+  file = File.expand_path(file)
+  file = file.sub(/\.(info|files)/,'')
+  $seen << file
+  Workflow.load_step file
+end
+
+step = get_step file
+$main_mtime = Open.exist?(step.path) ? Open.mtime(step.path) : nil
+
+data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
+
+jobs = step.rec_dependencies + [step]
+
+jobs = jobs.select{|job| job.info[:done]}.sort_by{|job| job.info[:started]}
+
+min_start = nil
+max_done = nil
+jobs.each do |job|
+  next unless job.done?
+  started = job.info[:started]
+  ddone = job.info[:done]
+  data[job.name] = [job.workflow.to_s, job.task_name,started,ddone]
+  if min_start.nil?
+    min_start = started
+  else
+    min_start = started if started < min_start
+  end
+
+  if max_done.nil?
+    max_done = ddone
+  else
+    max_done = ddone if ddone > max_done
+  end
+end
+
+#data.process "Start" do |value|
+#  value - min_start
+#end
+#
+#data.process "End" do |value|
+#  value - min_start
+#end
+
+Log.severity = 0
+data.R_interactive <<-EOF
+  rbbt.require('timevis')
+  data$id = rownames(data)
+  data$content = data$Task
+  data$start = data$Start
+  data$end = data$End
+
+  timevis(data)
+EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.25.45
+  version: 5.25.46
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-10 00:00:00.000000000 Z
+date: 2019-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -411,6 +411,7 @@ files:
 - share/rbbt_commands/workflow/remote/remove
 - share/rbbt_commands/workflow/server
 - share/rbbt_commands/workflow/task
+- share/rbbt_commands/workflow/trace
 - share/unicorn.rb
 - share/workflow_config.ru
 - test/rbbt/annotations/test_util.rb