rbbt-util 5.21.91 → 5.21.92

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f563cccf569fa61510d611058e9bbc3f15f4ecca
-  data.tar.gz: 8d11affe29abde8039c69e64c53206a40584bfd0
+  metadata.gz: '085baf05c5686c39e301773dce420dd46947b483'
+  data.tar.gz: 1e45580741e0644488f7c891f1d737d8ff421668
 SHA512:
-  metadata.gz: db6be501c52eefb3e8f7a66a494772462fb26652d3ec17f20215da24958f91d17ae5e7fb648c8dd12770cd500eced281fcd4fdbb174e1f9f86705cfd71f01bfe
-  data.tar.gz: ce353bf4de86ec04797f6cf6b839bb6ee7fb19e4ad67e21d481404d34ea253ab0f79e7d09fc436a3c476be27e9ca33618562f2faa8987acfacb3940f5d1f0c26
+  metadata.gz: b07ae9833c2832a79511ca6539d7de339e8008d83bc1d7056bb0d679355fce723b5233598c2ca372d0d55e99e7f0352c1c3f489f9c42dce0dc58ba89c3b99998
+  data.tar.gz: c41b15827ed605424506a7bf44ceebd27ad0c95012cc449cebb04d9e2660af6c96a6b9897f0ff8c9ccf91564bd196b4a5c0f66f43784dba12c1ae15e86d017eb

data/lib/rbbt/association/index.rb

@@ -100,6 +100,8 @@ module Association
 
         end
       end
+
+      data
     end.tap do |data|
       data.read if not Hash === data and data.respond_to? :read
       Association::Index.setup data

data/lib/rbbt/tsv/excel.rb

@@ -110,6 +110,7 @@ module TSV
           rows << row.values_at(0..(row.size - 1)).collect{|c| String === c ? c.gsub("\n", ' ') : c }
         end
 
+        num_values = rows.first.length
         File.open(filename, 'w') do |f|
           if header
             header = rows.shift
@@ -118,6 +119,7 @@ module TSV
 
           rows.each do |row| 
             values =  row.collect{|c| c.respond_to?(:value) ? c.value : c }
+            values[num_values-1] ||= nil
             f.puts values * "\t"
           end
         end
@@ -151,8 +153,6 @@ module TSV
       sheet = Misc.process_options options, :sheet
       header = Misc.process_options options, :header
 
-      iii options
-      iii sheet
       header = true unless header == false
       TmpFile.with_file do |filename|
         workbook = RubyXL::Parser.parse file
@@ -165,13 +165,17 @@ module TSV
           rows << row.cells.collect{|c| c.nil? ? nil : c.value}.collect{|c| String === c ? c.gsub("\n", ' ') : c }
         end
 
+        num_values = rows.first.length
         File.open(filename, 'w') do |f|
           if header
             header = rows.shift
             f.puts "#" + header * "\t"
           end
 
-          rows.each do |row| f.puts row * "\t" end
+          rows.each do |row| 
+            row[num_values-1] ||= nil
+            f.puts row * "\t" 
+          end
         end
 
         TSV.open(filename, options)

data/lib/rbbt/tsv/manipulate.rb

@@ -717,12 +717,12 @@ module TSV
 
   def transpose(key_field = "Unkown ID")
     case type
+    when :single, :flat
+      transpose_list self.to_list, key_field
     when :list
       transpose_list key_field
     when :double
       transpose_double key_field
-    else
-      raise "Transposing only works for TSVs of type :list or :double"
     end
   end
 

data/lib/rbbt/tsv/parser.rb

@@ -39,7 +39,7 @@ module TSV
 
       preamble << line if line
       while line and (TrueClass === @header_hash or (String === @header_hash and Misc.fixutf8(line) =~ /^#{@header_hash}/ ))
-        @fields = line.split(@sep)
+        @fields = line.split(@sep, -1)
         @key_field = @fields.shift
         @key_field = @key_field[(0 + header_hash.length)..-1] if String === @header_hash
 
@@ -234,7 +234,12 @@ module TSV
         if data.include? key
           new = data[key]
           new.each_with_index do |old, i|
-            old.concat values[i]
+            next if values[i].nil?
+            if old.nil?
+              new[i] = values[i] 
+            else
+              old.concat values[i]
+            end
           end
           data[key] = new
         else
@@ -247,6 +252,10 @@ module TSV
     def add_to_data_merge_zipped(data, keys, values)
       num = keys.length
 
+      values = values.collect do |v|
+        (v.nil? || v.empty?) ? [""] : v
+      end
+
       if values.first.length > 1 and num == 1
         keys = keys * values.first.length
         num = keys.length

data/lib/rbbt/tsv/stream.rb

@@ -248,7 +248,7 @@ module TSV
 
       while line  =~ /^#/
         if Hash === positions
-          new = (0..line.split(sep).length-1).to_a
+          new = (0..line.split(sep,-1).length-1).to_a
           positions.each do |k,v|
             new[k] = v
             new[v] = k
@@ -262,14 +262,14 @@ module TSV
 
       while line
         if Hash === positions
-          new = (0..line.split(sep).length-1).to_a
+          new = (0..line.split(sep, -1).length-1).to_a
           positions.each do |k,v|
             new[k] = v
             new[v] = k
           end
           positions = new
         end
-        values = line.split(sep)
+        values = line.split(sep, -1)
         new_values = values.values_at(*positions)
         sin.puts new_values * sep
         line = stream.gets

data/lib/rbbt/util/R.rb

@@ -156,6 +156,11 @@ module TSV
     script = require_sources + "\n\n" + script if require_sources
 
     r_options = Misc.pull_keys open_options, :R
+
+    r_options[:monitor] = open_options[:monitor] if open_options.include?(:monitor)
+    r_options[:method] = open_options[:method] if open_options.include?(:method)
+    r_options[:debug] = open_options[:debug] if open_options.include?(:debug)
+
     r_options[:debug] = true if r_options[:method] == :debug
     if r_options.delete :debug
       r_options[:monitor] = true

data/lib/rbbt/util/log.rb

@@ -17,7 +17,7 @@ module Log
   LOG_MUTEX = Mutex.new
 
   SEVERITY_NAMES ||= begin
-                     names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR ) 
+                     names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE ) 
                      names.each_with_index do |name,i|
                        eval "#{ name } = #{ i }" 
                      end
@@ -31,6 +31,7 @@ module Log
       line = stack.shift 
     end
     line ||= caller.first
+    line.gsub('`', "'")
   end
 
   def self._ignore_stderr
@@ -229,12 +230,12 @@ module Log
   def self.exception(e)
     stack = caller
     error([e.class.to_s, e.message].compact * ": " )
-    error("BACKTRACE: " << Log.last_caller(caller) << "\n" + color_stack(e.backtrace)*"\n")
+    error("BACKTRACE: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
   end
 
   def self.deprecated(m)
     stack = caller
-    warn("DEPRECATED: " << Log.last_caller(caller))
+    warn("DEPRECATED: " << Log.last_caller(stack))
     warn("* " << (m || "").to_s)
   end
 
@@ -248,7 +249,7 @@ module Log
   end
 
   def self.tsv(tsv)
-    STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller)
+    STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
     STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true) 
     STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
   end

data/lib/rbbt/util/misc/format.rb

@@ -181,4 +181,44 @@ module Misc
     end
   end
 
+  def self.parse_sql_values(txt)
+    io = StringIO.new txt.strip
+
+    values = []
+    fields = []
+    current = nil
+    quoted = false
+    while c = io.getc
+      if quoted
+        if c == "'"
+          quoted = false
+        else
+          current << c
+        end
+      else
+        case c
+        when "("
+          current = ""
+        when ")"
+          fields << current
+          values << fields
+          fields = []
+          current = nil
+        when ','
+          if not current.nil?
+            fields << current
+            current = ""
+          end
+        when "'"
+          quoted = true
+        when ";"
+          break
+        else
+          current << c
+        end
+      end
+    end
+    values
+  end
+
 end

data/lib/rbbt/util/misc/inspect.rb

@@ -66,6 +66,8 @@ module Misc
       (obj.respond_to?(:filename) and obj.filename ) ? "<IO:" + (obj.filename || obj.inspect + rand(100000)) + ">" : obj.inspect
     when File
       "<File:" + obj.path + ">"
+    when NamedArray
+      "[<NamedArray: fields=#{fingerprint obj.fields} -- values=#{fingerprint obj[0..-1]}]"
     when Array
       if (length = obj.length) > 10
         "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
@@ -289,4 +291,16 @@ module Misc
   def self.obj2md5(obj)
     obj2digest(obj)
   end
+
+  def self.get_filename(obj)
+    if obj.respond_to? :filename
+      obj.filename
+    elsif obj.respond_to? :path
+      obj.path
+    elsif (Path === obj || (String === obj && Misc.is_filename?(obj)))
+      obj
+    else
+      nil
+    end
+  end
 end

data/lib/rbbt/util/misc/omics.rb

@@ -308,7 +308,7 @@ module Misc
 
   def self.intersect_streams_read(io, sep=":")
     line = io.gets.chomp
-    parts = line.split(sep)
+    parts = line.split(sep, -1)
     chr, start, eend, *rest = parts
     start = start.to_i
     if eend =~ /^\d+$/

data/lib/rbbt/util/misc/pipes.rb

@@ -661,4 +661,30 @@ module Misc
   end
 
 
+  def self.swap_quoted_character(stream, charout="\n", charin=" ", quote='"')
+    io = Misc.open_pipe do |sin|
+      begin
+        quoted = false
+        prev = nil
+        while c = stream.getc
+          if c == quote and not prev == "\\"
+            quoted = ! quoted
+          end
+          c = charin if c == charout and quoted
+          sin << c
+          prev = c
+        end
+      rescue
+        stream.abort if stream.respond_to? :abort
+        raise $!
+      ensure
+        stream.join if stream.respond_to? :join
+      end
+    end
+  end
+
+  def self.remove_quoted_new_line(stream, quote = '"')
+    swap_quoted_character(stream, "\n", " ", quote)
+  end
+
 end

data/lib/rbbt/util/open.rb

@@ -9,7 +9,7 @@ module Open
   class OpenURLError < StandardError; end
   class OpenGzipError < StandardError; end
 
-  REMOTE_CACHEDIR = "/tmp/open_cache" 
+  REMOTE_CACHEDIR = File.join(ENV["HOME"], "/tmp/open_cache")
   FileUtils.mkdir REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
 
   class << self
@@ -126,11 +126,11 @@ module Open
   # Grep
   
   def self.grep(stream, grep, invert = false)
-    grep_cmd = 'egrep'
+    grep_cmd = ENV["GREP_CMD"] || "/bin/grep"
     case 
     when Array === grep
       TmpFile.with_file(grep * "\n", false) do |f|
-        CMD.cmd("#{grep_cmd} #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+        CMD.cmd("#{grep_cmd} #{invert ? '-v' : ''} -", "-w" => true, "-F" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
       end
     else
       CMD.cmd("#{grep_cmd} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
@@ -319,7 +319,7 @@ module Open
   end
 
   def self.zip?(file)
-    !! (file =~ /\.zip/)
+    !! (file =~ /\.zip$/)
   end
 
 

data/lib/rbbt/workflow.rb

@@ -386,6 +386,7 @@ module Workflow
     task = task_for path
     return remote_tasks[task].load_id(id) if remote_tasks and remote_tasks.include? task
     step = Step.new path, tasks[task.to_sym]
+    step.load_inputs_from_info
     set_step_dependencies(step)
     step
   end

data/lib/rbbt/workflow/accessor.rb

@@ -696,7 +696,7 @@ module Workflow
       when :compute
         compute = v
       when Symbol
-        rec_dependency = all_d.select{|d| d.task_name.to_sym == v }.first
+        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
 
         if rec_dependency.nil?
           if inputs.include? v
@@ -734,6 +734,8 @@ module Workflow
     path_deps = {}
     dependencies.each do |dependency|
       _inputs = IndiferentHash.setup(inputs.dup)
+      jobname = _inputs[:jobname] if _inputs.include? :jobname
+
       real_dep = case dependency
                  when Array
                    workflow, dep_task, options = dependency
@@ -743,6 +745,7 @@ module Workflow
                    all_d = (real_dependencies + real_dependencies.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
                    
                    _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
+                   jobname = _inputs[:jobname] if _inputs.include? :jobname
 
                    job = workflow.job(dep_task, jobname, _inputs)
                    ComputeDependency.setup(job, compute) if compute

data/lib/rbbt/workflow/step.rb

@@ -45,25 +45,44 @@ class Step
                     end
     @mutex = Mutex.new
     @info_mutex = Mutex.new
-    @inputs = inputs || []
+    @inputs = inputs 
     NamedArray.setup @inputs, task.inputs.collect{|s| s.to_s} if task and task.respond_to? :inputs and task.inputs
   end
 
-  def inputs
-    if @inputs.nil? and task and task.respond_to? :inputs
-      @inputs = info[:inputs].values_at *task.inputs.collect{|name| name.to_s}
-    end
+  def workflow
+    info[:workflow]
+  end
 
-    if task.inputs and not NamedArray === @inputs
-      NamedArray.setup @inputs, task.inputs 
+  def load_inputs_from_info
+    if info[:inputs]
+      info_inputs = info[:inputs]
+      if task && task.respond_to?(:inputs) && task.inputs
+        IndiferentHash.setup info_inputs
+        @inputs = NamedArray.setup info_inputs.values_at(*task.inputs.collect{|name| name.to_s}), task.inputs
+      else
+        @inputs = NamedArray.setup info_inputs.values, info_inputs.keys
+      end
+    else
+      nil
     end
+  end
+
+  def inputs
+    return @inputs if NamedArray === @inputs
+
+    load_inputs_from_info if @inputs.nil? 
 
-    @inputs
+    NamedArray.setup(@inputs, task.inputs) if task && task.inputs && ! NamedArray === @inputs
+
+    @inputs || []
   end
 
   def recursive_inputs
     if NamedArray === inputs
-      i = Hash[*inputs.fields.zip(inputs).flatten]
+      i = {}
+      inputs.zip(inputs.fields).each do |v,f|
+        i[f] = v
+      end
     else
       i = {}
     end
@@ -113,11 +132,27 @@ class Step
     self
   end
 
+  def result_type
+    @result_type ||= if @task.nil?
+                       info[:result_type]
+                     else
+                       @task.result_type
+                     end
+  end
+
+  def result_description
+    @result_description ||= if @task.nil?
+                       info[:result_description]
+                     else
+                       @task.result_description
+                     end
+  end
+
   def prepare_result(value, description = nil, entity_info = nil)
     res = case 
     when IO === value
       begin
-        res = case @task.result_type
+        res = case result_type
               when :array
                 array = []
                 while line = value.gets
@@ -206,13 +241,13 @@ class Step
             res = @result
           else
             join if not done?
-            @path.exists? ? Persist.load_file(@path, @task.result_type) : exec
+            @path.exists? ? Persist.load_file(@path, result_type) : exec
           end
 
-    if @task.result_description
+    if result_description
       entity_info = info.dup
       entity_info.merge! info[:inputs] if info[:inputs]
-      res = prepare_result res, @task.result_description, entity_info 
+      res = prepare_result res, result_description, entity_info 
     end
 
     res

data/lib/rbbt/workflow/step/dependencies.rb

@@ -164,7 +164,7 @@ class Step
       raise $!
     rescue Exception
       Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s } -- #{$!.message}"
-      Log.exception $!
+      #Log.exception $!
       raise $!
     end
   end

data/lib/rbbt/workflow/step/run.rb

@@ -2,7 +2,7 @@ require 'rbbt/workflow/step/dependencies'
 
 class Step
 
-  attr_reader :stream, :dupped, :saved_stream, :inputs
+  attr_reader :stream, :dupped, :saved_stream
 
   def get_stream
     @mutex.synchronize do
@@ -21,7 +21,7 @@ class Step
   def resolve_input_steps
     step = false
     pos = 0
-    new_inputs = @inputs.collect do |i| 
+    new_inputs = inputs.collect do |i| 
       begin
         if Step === i
           step = true
@@ -115,6 +115,10 @@ class Step
             :issued => (issue_time = Time.now),
             :name => name,
             :clean_name => clean_name,
+            :workflow => @task.workflow.to_s,
+            :task_name => @task.name,
+            :result_type => @task.result_type,
+            :result_description => @task.result_description
           })
 
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
@@ -124,11 +128,19 @@ class Step
           rescue Exception
             FileUtils.rm pid_file if File.exist?(pid_file)
             stop_dependencies
-            Log.exception $!
             raise $!
           end
 
-          set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
+          if not task.inputs.nil?
+            info_inputs = @inputs.collect do |i| 
+              if Path === i 
+                i.to_s
+              else 
+                i 
+              end
+            end
+            set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, info_inputs)) 
+          end
 
           set_info :started, (start_time = Time.now)
           log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"

data/lib/rbbt/workflow/task.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
 require 'rbbt/persist'
 
 module Task
-  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension
+  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow
 
   def self.setup(options = {}, &block)
     block.extend Task
@@ -92,6 +92,8 @@ module Task
       raise "Dependency task not found: #{dep}" if task.nil?
       next if seen.include? [wf, task.name]
 
+      task.workflow = wf if wf
+
       seen << [wf, task.name]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?

data/lib/rbbt/workflow/usage.rb

@@ -33,7 +33,11 @@ module Task
       task_inputs = dep_inputs deps, workflow
       task_inputs.each do |task,new_inputs|
         task.inputs.zip(task.input_types.values_at(*task.inputs)).select{|i,t| t.to_sym == :select and task.input_options[i][:select_options] }.each{|i,t| selects << [i, task.input_options[i][:select_options]]  }
-        puts "  #{Log.color :yellow, task.name.to_s}:"
+        if task.workflow and task.workflow != workflow
+          puts "  #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
+        else
+          puts "  #{Log.color :yellow, task.name.to_s}:"
+        end
         puts
         puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
         puts

data/share/Rlib/util.R

@@ -472,12 +472,12 @@ rbbt.get.modes <- function(x,bw = NULL,spar = NULL) {
 
 #{{{ PLOTS
 
-rbbt.png_plot <- function(filename, width, height, p, ...){
+rbbt.png_plot <- function(filename, p, width=500, height=500, ...){
     png(filename=filename, width=width, height=height, ...);
     eval(parse(text=p));
 }
 
-rbbt.heatmap <- function(filename, width, height, data, take_log=FALSE, ...){
+rbbt.heatmap <- function(filename, data, width=500, height=500, take_log=FALSE, ...){
     opar = par()
     png(filename=filename, width=width, height=height);
 
@@ -630,7 +630,35 @@ rbbt.plot.text_scatter <- function(formula, data) {
     text(formula, data=data, cex = 0.6, labels=rownames(data))
 }
 
-install.bioc <-function(pkg){
-    source("http://bioconductor.org/biocLite.R")
-    biocLite(pkg)
+rbbt.install.CRAN <- function(pkg){
+    res = FALSE
+    tryCatch({ install.packages(pkg); res = TRUE }, error = function(e){ warning(paste("Could not install CRAN ", pkg)); res = FALSE })
+    return(res)
 }
+rbbt.install.bioc <-function(pkg){
+    res = FALSE
+    tryCatch({ source("http://bioconductor.org/biocLite.R"); biocLite(pkg); res = TRUE }, error = function(e){ warning(paste("Could not install Bioconductor ", pkg)); res = FALSE })
+    return(res)
+}
+
+rbbt.install.github <- function(pkg, ...){
+    res = FALSE
+    tryCatch({ library(devtools); install_github(pkg, ...); res = TRUE }, error = function(e){ warning(paste("Could not install GITHUB ", pkg)); res = FALSE })
+    return(res)
+}
+
+rbbt.require <- function(pkg, ...){
+    list.of.packages <- c(pkg)
+    new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
+    for (pkg in new.packages){ 
+        if (!rbbt.install.github(pkg, ...)){
+            if (!rbbt.install.CRAN(pkg, ...)){
+                if (!rbbt.install.bioc(pkg, ...)){
+                    stop("Error!", pkg)
+                }
+            }
+        }
+    }
+    library(list.of.packages, character.only=T)
+}
+

data/share/rbbt_commands/app/template

@@ -13,7 +13,7 @@ require 'rbbt/rest/knowledge_base'
 require 'rbbt/rest/helpers'
 require 'rbbt/rest/web_tool'
 
-options = SOPT.get "-e--environment*:-p--port*:-s--server*:-b--bind*:-e--environment*:-R--RServe_session*:--finder:--views*:-w--workflows*"
+options = SOPT.get "-e--environment*:-p--port*:-s--server*:-b--bind*:-e--environment*:-R--RServe_session*:--finder:--views*:-W--workflows*"
 
 template = ARGV.first
 

data/share/rbbt_commands/log

@@ -18,6 +18,7 @@ HIGH
 INFO
 WARN
 ERROR
+NONE
 
 -h--help Print this help
 

data/share/rbbt_commands/tsv/get

@@ -3,10 +3,29 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 
-options = SOPT.get("-tch--tokyocabinet:-tcb--tokyocabinet_bd")
+options = SOPT.setup <<EOF
 
-file = ARGV.shift
-value = ARGV.shift
+Query a TSV value
+
+$ rbbt tsv get [options] <filename.tsv|-> <key>
+
+Use - to read from STDIN
+
+-tch--tokyocabinet File is a tokyocabinet hash database
+-tcb--tokyocabinet_bd File is a tokyocabinet B database
+-f--fields* Fields to extract
+-k--key_field* Use this field as key
+-h--help Print this help
+-l--lines Separate in lines
+EOF
+
+rbbt_usage and exit 0 if options[:help]
+
+file, key = ARGV
+
+raise ParameterException, "Please specify file and key" if key.nil?
+
+file = STDIN if file == '-'
 
 case
 when options[:tokyocabinet]
@@ -14,13 +33,31 @@ when options[:tokyocabinet]
 when options[:tokyocabinet_bd]
   tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
 else
-  tsv = TSV.open(file, :grep => value )
+  if String === file
+    file = file.dup
+    Path.setup(File.expand_path(file))
+  end
+  tsv = file
 end
 
-res = tsv[value]
+fields = options[:fields]
+key_field = options[:key_field]
+fields = fields.split(/[,|]/, -1) unless fields.nil?
 
-if res.nil?
-  raise "RECORD NOT FOUND"
-else
-  puts tsv[value].report
+parser = TSV::Parser.new tsv, :key_field => key_field,  :fields => fields
+fields ||= parser.fields
 
+TSV.traverse(parser) do |k,v|
+  next unless k.include? key
+  if fields.length == 1
+    if options[:lines]
+      puts (Array === v ? v.flatten*"\n" : v.to_s ) 
+    else
+      puts (Array === v ? v.flatten*"\t" : v.to_s ) 
+    end
+  else
+    fields.zip(v).each do |field,v|
+      puts "#{Log.color :magenta, field+":"} #{v}"
+    end
+  end
+end

data/share/rbbt_commands/tsv/info

@@ -63,7 +63,7 @@ else
   end
     puts "Rows: #{Log.color :blue, rows}" 
     parts = []
-    header.first_line.split(header.sep).each_with_index{|p,i| parts << (Log.color(:cyan, "(#{i}) ") << p.strip) }
+    header.first_line.split(header.sep, -1).each_with_index{|p,i| parts << (Log.color(:cyan, "(#{i}) ") << p.strip) }
     puts parts * "\t"
   puts
 end

data/share/rbbt_commands/tsv/values

@@ -14,6 +14,7 @@ Use - to read from STDIN
 -tch--tokyocabinet File is a tokyocabinet hash database
 -tcb--tokyocabinet_bd File is a tokyocabinet B database
 -f--field* Limit to a particular field
+-k--keys Print also keys
 -h--help Print this help
 -l--lines Separate in lines
 
@@ -45,6 +46,8 @@ fields = field.nil? ? nil : [field]
 TSV.traverse(tsv, :fields => fields) do |k,v|
   if options[:lines]
     puts (Array === v ? v.flatten*"\n" : v.to_s ) 
+  elsif options[:keys]
+    puts(k << "\t" << (Array === v ? v.flatten*"\t" : v.to_s ))
   else
     puts (Array === v ? v.flatten*"\t" : v.to_s ) 
   end

data/share/rbbt_commands/workflow/info

@@ -132,6 +132,7 @@ if recursive
 
   while deps.any? do
     dep = deps.shift
+    inputs = {} if inputs.nil?
     inputs = inputs.merge(dep.info[:inputs] || {})
     deps.concat (dep.info[:dependencies] || []).collect{|v| get_step v.last }
   end
@@ -139,14 +140,16 @@ if recursive
   inputs.each do |input,value|
     case value
     when nil
-      puts "  " << Misc.format_definition_list_item(input, 'nil', 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " << input.to_s, 'nil', 80, 20, :blue)
     when Array
-      puts "  " << Misc.format_definition_list_item(input, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), 80, 20, :blue).gsub("\n\n","\n")
     when TrueClass, FalseClass
-      puts "  " << Misc.format_definition_list_item(input, value.to_s, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, 80, 20, :blue)
     else
-      text = value.to_s.split("\n")[0..5].compact * "\n\n"
-      puts "  " << Misc.format_definition_list_item(input, text, 80, 20, :blue)
+      lines = value.to_s.split("\n").collect{|l| l.length >= 60 ? l[0..45] + " ..." : l }
+      text = lines[0..5].compact * "\n\n"
+      text << "\n\n...\n\n" if lines.length > 6
+      puts Misc.format_definition_list_item("  " << input.to_s, text, 80, 20, :blue).gsub("\n\n","\n")
     end
   end
 end

data/share/rbbt_commands/workflow/task

@@ -184,6 +184,7 @@ the job dependencies recursively.
 -pn--printname Print the name of the job and exit without starting it
 -pf--printpath Print the path of the job result
 -cl--clean Clean the last step of the job so that it gets recomputed
+-ct--clean_task* Clean a particular dependency task
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
 --detach Run job asyncronously and detach process
@@ -210,6 +211,7 @@ do_fork = !!options.delete(:fork)
 detach = !!options.delete(:detach)
 do_exec = !!options.delete(:exec)
 clean = !!options.delete(:clean)
+clean_task = options.delete(:clean_task)
 recursive_clean = !!options.delete(:recursive_clean)
 out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT
 
@@ -298,6 +300,12 @@ if clean
   sleep 1
 end
 
+if clean_task
+  job.rec_dependencies.each do |dep|
+    dep.clean if dep.task_name.to_s == clean_task.to_s
+  end
+end
+
 if recursive_clean 
   job.recursive_clean 
 end

data/test/rbbt/util/test_misc.rb

@@ -557,4 +557,11 @@ eum fugiat quo voluptas nulla pariatur?"
     assert Misc.break_lines(text, 10).split("\n").length == 1 + text.length / 10
     assert Misc.break_lines(text, 10).split("\n").select{|l| l.length > 10 }.empty?
   end
+
+  def test_parse_sql_values
+    str=<<'EOF'
+(xxx,yyy,zzz),(aaa,'bb(,)b',ccc)
+EOF
+    assert Misc.parse_sql_values(str)[1][1] == "bb(,)b"
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.91
+  version: 5.21.92
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-06 00:00:00.000000000 Z
+date: 2017-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -508,83 +508,83 @@ signing_key:
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/test_workflow.rb
+- test/test_helper.rb
 - test/rbbt/resource/test_path.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_log.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_R.rb
 - test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
-- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/test_concurrency.rb
-- test/rbbt/util/test_R.rb
-- test/rbbt/util/log/test_progress.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/test_packed_index.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_association.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/test_resource.rb
 - test/rbbt/test_entity.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/test_tsv.rb
 - test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_task.rb
-- test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_doc.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/test_persist.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/tsv/test_field_index.rb
+- test/rbbt/workflow/test_step.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/test_association.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/parallel/test_through.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/test_matrix.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_parser.rb
-- test/rbbt/tsv/test_stream.rb
-- test/rbbt/tsv/test_util.rb
 - test/rbbt/tsv/test_excel.rb
 - test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_stream.rb
 - test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/parallel/test_through.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_field_index.rb
+- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_parser.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/test_persist.rb
 - test/rbbt/test_fix_width_table.rb
-- test/test_helper.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb