rbbt-util 5.6.6 → 5.6.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dfcdf5fe55d3a3ddb6a46648fd5a7d378b2eb692
-  data.tar.gz: f7a1a8275c767cb3347e3ac8856be2353014f4f3
+  metadata.gz: d05f8baf821976ddb0420b00980293a3cec0f400
+  data.tar.gz: 13794d29480143f51eb5438dbd92426ef298a22f
 SHA512:
-  metadata.gz: 2caa0e4bb980ef419ba0390c2d2b34eec883e88f71e24979e837b3e2720198f5ea7a1dc053583fa0941a8e3071d3a31e7db897fd0fec03cf8f64664d3e119924
-  data.tar.gz: c51c7c567113673fca18fa95dae7bf2722e6a0d0bdd75f5236fa4fd2a89109bfa72a3fc648d212540ebf4c2d89e4f5435afdd01a66ca9b060e44a386aa82afce
+  metadata.gz: 1f4a308e54ad217ab01b4b1930b1c07de7ccfddefbbe13c96b6d19c0d14227a07f1cfbbe0e3db6db745123a3b7c332b8f7071cbbc28210401d805eec2ab63cdf
+  data.tar.gz: ed3a3dadb218c3a979f05cc7a248657f561d17e48e165e2d187620f468a99ac3a04f924d6f11c55a98a50f5e77a144a9844263ff66e3debfa7c5035081155283

data/bin/rbbt

@@ -3,7 +3,10 @@
 require 'rbbt'
 require 'rbbt/util/simpleopt'
 
-options = SOPT.get("--log* Log level from 0 (debug) 6 (errors):-cd--command_dir* Directory from where to load commands:--profile Profile execution")
+options = SOPT.get("--log* Log level from 0 (debug) 6 (errors):-cd--command_dir* Directory from where to load commands:--profile Profile execution:--no_color Disable colored output:--locate_file Locate file been executed")
+
+ENV["RBBT_NOCOLOR"] = "true" if options[:no_color]
+locate = options.delete :locate_file
 
 if options[:log]
   Log.severity = options[:log].to_i
@@ -47,13 +50,14 @@ def rbbt_usage(prev = nil)
   puts SOPT.doc
 
   if prev
-    puts "## COMMANDS"
     puts
-    puts "Command:"
+    puts Log.color :magenta, "## COMMANDS"
+    puts
+    puts Log.color :magenta, "Command:"
     puts 
     puts "  #{File.basename($0)} #{prev * " "}"
     puts 
-    puts "Subcommands:"
+    puts Log.color :magenta, "Subcommands:"
     puts 
     commands(prev).each do |command|
       puts "  " << command
@@ -86,8 +90,13 @@ begin
       prev << command
       dir = dir[command]
     when dir[command].exists?
-      load dir[command].find
-      exit 0
+      if locate
+        puts dir[command].find
+        exit 0
+      else
+        load dir[command].find
+        exit 0
+      end
     else
       error = "Command '#{command }' not understood"
       rbbt_usage(prev)

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -76,6 +76,13 @@ module Persist
       @writable
     end
 
+    #def each
+    #  iterinit
+    #  while key = iternext
+    #    yield key, get(key)
+    #  end
+    #end
+
     def collect
       res = []
       each do |key, value|

data/lib/rbbt/tsv.rb

@@ -13,6 +13,7 @@ require 'rbbt/tsv/manipulate'
 require 'rbbt/tsv/index'
 require 'rbbt/tsv/attach'
 require 'rbbt/tsv/filter'
+require 'rbbt/tsv/field_index'
 
 module TSV
   class << self

data/lib/rbbt/tsv/accessor.rb

@@ -8,7 +8,7 @@ module TSV
   attr_accessor :unnamed, :serializer_module, :entity_options, :entity_templates
 
   def annotate(tsv)
-    TSV.setup(tsv, :key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers)
+    TSV.setup(tsv, :key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers, :unnamed => unnamed)
   end
 
   def entity_options

data/lib/rbbt/tsv/field_index.rb

@@ -0,0 +1,56 @@
+
+module TSV
+  class << self
+    attr_accessor :field_index_dir
+    def field_index_dir
+      @field_index_dir ||= Rbbt.var.cache.field_indices
+    end
+  end
+
+  attr_accessor :field_indices
+
+
+  def field_index(field)
+    @field_indices ||= {}
+    @field_indices[field] ||= Persist.persist_tsv(self, filename, {:field => field}, :prefix => "FieldIndex", :dir => TSV.field_index_dir, :persist => true, :serializer => :list, :engine => "BDB" ) do |data|
+      tsv = {}
+      case type 
+      when :single, :list
+        through :key, [field] do |key, values|
+          value = values.first
+          tsv[value] ||= []
+          tsv[value] << key
+        end
+      else
+        through :key, [field] do |key, values|
+          values.first.each do |value|
+            tsv[value] ||= []
+            tsv[value] << key
+          end
+        end
+      end
+
+      tsv.each do |v,keys|
+        data[v] = keys.sort
+      end
+
+      data
+    end
+  end
+
+  def field_index_select(matches)
+    final = nil
+    matches.each do |field,values|
+      i = field_index(field)
+
+      if Array === values
+        keys = values.inject([]){|acc,value| m = i[value]; acc = m.nil? ? acc : Misc.merge_sorted_arrays( acc, m) }
+      else
+        keys = i[values] || []
+      end
+
+      final = final.nil? ? keys : Misc.intersect_sorted_arrays(final, keys)
+    end
+    final
+  end
+end

data/lib/rbbt/tsv/filter.rb

@@ -57,6 +57,8 @@ module Filtered
             value == @value or (Array === value and value.include? @value)
           end
         EOC
+      else
+        raise "Unknown match: #{ @match }"
       end
     end
 

data/lib/rbbt/tsv/manipulate.rb

@@ -276,12 +276,10 @@ module TSV
         end
 
         data.extend TSV unless TSV === data
+        self.annotate(data)
+
         data.key_field = new_key_field_name
         data.fields = new_field_names
-        data.filename = filename
-        data.namespace = namespace
-        data.entity_options = entity_options
-        data.entity_templates = {}
         data.fields.each do |field|
           data.entity_templates[field] = entity_templates[field] if entity_templates.include? field
         end

data/lib/rbbt/util/log.rb

@@ -23,12 +23,31 @@ module Log
 
   WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
 
-  SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
+  if ENV["RBBT_NOCOLOR"] == "true"
+    SEVERITY_COLOR = [reset, "", "", "", "", "", ""] #.collect{|e| "\033[#{e}"}
+  else
+    SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
+  end
 
   HIGHLIGHT = "\033[1m"
 
-  def self.color(severity)
-    SEVERITY_COLOR[severity]
+  def self.color(severity, str = nil)
+    return str || "" if ENV["RBBT_NOCOLOR"] == "true"
+    color = SEVERITY_COLOR[severity] if Fixnum === severity
+    color = Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity 
+    if str.nil?
+      color
+    else
+      color + str + color(0)
+    end
+  end
+
+  def self.highlight(str = nil)
+    if str.nil?
+      HIGHLIGHT
+    else
+      HIGHLIGHT + str + color(0)
+    end
   end
 
   def self.log(message = nil, severity = MEDIUM, &block)
@@ -40,8 +59,8 @@ module Log
 
     sev_str = severity.to_s
 
-    prefix = time << "[" << SEVERITY_COLOR[severity] << sev_str << SEVERITY_COLOR[0] << "]"
-    message = "" << HIGHLIGHT << message << SEVERITY_COLOR[0] if severity >= INFO
+    prefix = time << "[" << color(severity) << sev_str << color(0) << "]"
+    message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message
 
     STDERR.puts str

data/lib/rbbt/util/simpleopt/doc.rb

@@ -26,7 +26,8 @@ module SOPT
   end
 
   def self.input_format(name, type = nil, default = nil, short = nil)
-    input_str = (short.nil? or short.empty?) ? "--#{name}" : "-#{short}, --#{name}"
+    input_str = (short.nil? or short.empty?) ? Log.color(:blue,"--#{name}") : Log.color(:blue, "-#{short}") << ", " << Log.color(:blue, "--#{name}")
+    input_str = Log.color(:blue, input_str)
     input_str << case type
     when nil
       "#{default != nil ? " (default '#{default}')" : ""}:"
@@ -71,18 +72,20 @@ module SOPT
 
   def self.doc
     doc = <<-EOF
+#{Log.color :magenta}
 #{command}(1) -- #{summary}
 #{"=" * (command.length + summary.length + 7)}
+#{Log.color :reset}
 
-## SYNOPSYS
+#{ Log.color :magenta, "## SYNOPSYS"}
 
 #{synopsys}
 
-## DESCRIPTION
+#{ Log.color :magenta, "## DESCRIPTION"}
 
 #{description}
 
-## OPTIONS
+#{ Log.color :magenta, "## OPTIONS"}
 
 #{input_doc(inputs, input_types, input_descriptions, input_defaults, input_shortcuts)}
     EOF

data/lib/rbbt/workflow.rb

@@ -70,22 +70,6 @@ module Workflow
       else
         filename = workflow_dir[wf_name]['workflow.rb'].find
       end
-
-      #case
-      #  # Points to workflow file
-      #when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb'))
-      #  filename = (wf_name =~ /\.?\//) ? wf_name : "./" << wf_name 
-      #when (defined?(Rbbt) and Rbbt.etc.workflow_dir.exists?)
-      #  dir = Rbbt.etc.workflow_dir.read.strip
-      #  dir = File.join(dir, wf_name)
-      #  filename = File.join(dir, 'workflow.rb')
-      #when defined?(Rbbt)
-      #  path = Rbbt.workflows[wf_name].find
-      #  filename = File.join(path, 'workflow.rb')
-      #else
-      #  path = File.join(ENV['HOME'], '.workflows', wf_name)
-      #  filename = File.join(dir, 'workflow.rb')
-      #end
     end
 
     if filename and File.exists? filename
@@ -99,9 +83,9 @@ module Workflow
 
     # Already loaded
     begin
-      Misc.string2const wf_name
+      workflow = Misc.string2const wf_name
       Log.debug{"Workflow #{ wf_name } already loaded"}
-      return true
+      return workflow
     rescue Exception
     end
 
@@ -129,6 +113,11 @@ module Workflow
     require_local_workflow(Misc.snake_case(wf_name)) or 
     (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)}` and require_local_workflow(Misc.snake_case(wf_name))) or
     raise("Workflow not found or could not be loaded: #{ wf_name }")
+    begin
+      Misc.string2const Misc.camel_case(wf_name)
+    rescue
+      Workflow.workflows.last || true
+    end
   end
 
   attr_accessor :description

data/lib/rbbt/workflow/accessor.rb

@@ -347,7 +347,9 @@ module Workflow
       workdir_find = workdir
     end
  
-    Misc.path_relative_to(workdir_find, File.dirname(path)).sub(/([^\/]+)\/.*/,'\1')
+    workdir_find = File.expand_path(workdir_find)
+    path = File.expand_path(path)
+    dir = File.dirname(path)
+    Misc.path_relative_to(workdir_find, dir).sub(/([^\/]+)\/.*/,'\1')
   end
-
 end

data/lib/rbbt/workflow/usage.rb

@@ -2,18 +2,20 @@ require 'rbbt/util/simpleopt'
 
 module Task
   def doc(deps = nil)
-    puts "## #{ name }:"
-    puts "\n" << description if description and not description.empty?
+    puts Log.color :magenta, "## #{ name }:"
+    puts "\n" << description  << "\n" if description and not description.empty?
+    puts "Returns: " << Log.color(:blue, result_type.to_s) << "\n"
     puts SOPT.input_doc(inputs, input_types, input_descriptions, input_defaults, true)
+      puts
+
 
     if deps and deps.any?
-      puts
       puts "From dependencies:"
       puts
       deps.each do |dep|
         puts "  #{dep.name}:"
         puts
-        puts SOPT.input_doc(dep.inputs, dep.input_types, dep.input_descriptions, dep.input_defaults, true)
+        puts SOPT.input_doc((dep.inputs - self.inputs), dep.input_types, dep.input_descriptions, dep.input_defaults, true)
         puts
       end
     end
@@ -24,17 +26,17 @@ module Workflow
   def doc(task = nil)
 
     if task.nil?
-      puts self.to_s 
-      puts "=" * self.to_s.length
+      puts Log.color :magenta, self.to_s 
+      puts Log.color :magenta, "=" * self.to_s.length
       puts
       puts "\n" << workflow_description if workflow_description and not workflow_description.empty?
       puts
 
-      puts "## TASKS"
+      puts Log.color :magenta, "## TASKS"
       puts
       tasks.each do |name,task|
-        puts "  * #{ name }:"
-        puts "    " << task.description if task.description and not task.description.empty?
+        puts "  * #{ Log.color :green, name.to_s }:"
+        puts "    " << task.description.split(/\n\s*\n/).first if task.description and not task.description.empty?
         puts
       end
     else

data/share/rbbt_commands/workflow/list

@@ -12,16 +12,17 @@ Path.setup(workflow_dir) unless Path === workflow_dir
 
 files = workflow_dir.find_all.collect{|p| p.glob("*") }.flatten.select{|f| File.directory? f }
 
-puts defined? COSMIC
 if options[:describe]
   files.each do |file|
     workflow = File.basename(file)
     Workflow.require_workflow workflow
-    ddd Workflow.workflows
     workflow = Workflow.workflows.select{|w| Misc.camel_case(w.to_s) == Misc.camel_case(workflow)}.first
-    ddd workflow
-    puts [Misc.camel_case(workflow.to_s), workflow.description] * ":"
-    puts defined? COSMIC
+    puts "# "<<[Misc.camel_case(workflow.to_s), workflow.description] * ": "
+    workflow.tasks.each do |name,task|
+      puts "  * " << [name, task.inputs * " "] * ": "
+    end
+    puts
+
   end
 else
   files.each do |file|

data/share/rbbt_commands/workflow/provenance

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/workflow'
+
+
+
+require 'rbbt-util'
+require 'fileutils'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow/step'
+require 'rbbt/util/misc'
+
+$workdir = Path.setup(File.expand_path(ARGV[0] || '.'))
+
+
+YAML::ENGINE.yamler = 'syck' if defined? YAML::ENGINE and YAML::ENGINE.respond_to? :yamler
+
+def info_files
+  $workdir.glob(Step.info_file('**/*'))
+end
+
+def running?(info)
+  Misc.pid_exists? info[:pid]
+end
+
+def job_str(file, info, severity_color = nil)
+  clean_file = file.sub('./','').sub('.info','').gsub('/', Term::ANSIColor.blue(' => '))
+  if $name
+    clean_file
+  else
+    info ||= {:status => :missing_info_file}
+    str = [clean_file, info[:status].to_s] * "\t\t\t[ STATUS = " + " ]" 
+    if info[:status] != :error and info[:status] != :aborted and not info[:status] == :done
+      str += " (#{running?(info)? :running : :dead} #{info[:pid]})" if info[:pid]
+      str += " (children: #{info[:children_pids].collect{|pid| [pid, Misc.pid_exists?(pid) ? "R" : "D"] * ":"} * ", "})" if info.include? :children_pids
+    end
+
+    if severity_color
+      "#{severity_color}" <<  str  << "\033[0m" 
+    else
+      str
+    end
+  end
+end
+
+file = ARGV[0]
+
+deps = {}
+
+$done = []
+def print(file, offset = 1)
+  if not $done.include? file
+    putc "*"
+  end
+
+  step = Step.new file.sub('.info','')
+  
+  info = step.info
+  dependencies = info[:dependencies] || []
+  color = case
+          when (not info)
+            Log::SEVERITY_COLOR[3]
+          when info[:status] == :error 
+            Log::SEVERITY_COLOR[3]
+          when info[:status] == :aborted 
+            Log::SEVERITY_COLOR[2]
+          when (info[:status] != :done and info[:pid] and not running? info)
+            Log::SEVERITY_COLOR[2]
+          end
+
+  puts (" " * offset * 2) << job_str(step.path, step.info, color)
+  $done << file
+  root = File.dirname(File.dirname(file))
+  dependencies.each do |task,new_file|
+    new_file = File.join(root, task.to_s, new_file)
+    print(new_file, offset+1)
+  end
+end
+
+print(file)

data/share/rbbt_commands/workflow/task

@@ -6,9 +6,25 @@ require 'rbbt/workflow/usage'
 
 YAML::ENGINE.yamler = 'syck' if defined? YAML::ENGINE and YAML::ENGINE.respond_to? :yamler
 
+def load_inputs(dir, task)
+  inputs = {}
+  dir = Path.setup(dir.dup)
+  task.inputs.each do |input|
+    file = dir[input].find
+    next unless file.exists?
+    case task.input_types[input]
+    when :tsv, :array, :text
+      inputs[input.to_sym]  = file
+    else
+      inputs[input.to_sym]  = file.read
+    end
+  end
+  inputs
+end
+
 def usage(workflow = nil, task = nil, exception=nil)
   puts SOPT.doc
-  puts "## WORKFLOW"
+  puts Log.color :magenta, "## WORKFLOW"
   puts
   if workflow.nil?
     puts "No workflow specified"
@@ -19,8 +35,8 @@ def usage(workflow = nil, task = nil, exception=nil)
     workflow.load_tasks if workflow.respond_to? :load_tasks
     workflow.doc
   else
-    puts workflow.to_s
-    puts "=" * workflow.to_s.length
+    puts Log.color :magenta, workflow.to_s
+    puts Log.color :magenta, "=" * workflow.to_s.length
     puts
     puts workflow.workflow_description
     puts
@@ -115,6 +131,7 @@ options = SOPT.get <<EOF
 -O--output* Save job result into file:
 -jf--job_file* Output one of the job produced files:
 -ljf--list_job_files List all the files produced in that step:
+--load_inputs* Load inputs from a directory
 --info Show the job info:
 --provenance Report the jobs provenance:
 EOF
@@ -173,7 +190,11 @@ name = options.delete(:jobname) || "Default"
 
 # get job args
 sopt_option_string = SOPT_options(workflow, task)
-job_options = SOPT.get sopt_option_string
+if options[:load_inputs]
+  job_options = load_inputs(options[:load_inputs], task)
+else
+  job_options = SOPT.get sopt_option_string
+end
 job_options = fix_options(workflow, task, job_options)
 
 workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir))) if options[:workdir]

data/test/rbbt/tsv/test_field_index.rb

@@ -0,0 +1,19 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/field_index'
+
+class TestTSVFieldIndex < Test::Unit::TestCase
+
+  def test_zipped
+    content =<<-EOF
+#Id    ValueA    ValueB ValueC
+rowA    A|AA    B|BB  C|CC
+rowa    a|aa    b|BB  C|CC
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :double)
+      puts tsv.field_index("ValueA")
+    end
+  end
+end

data/test/rbbt/tsv/test_filter.rb

@@ -54,6 +54,7 @@ row3    A    C
       elem = []
       tsv.through do |k,v| elem << k end
       assert_equal 2, elem.size
+      assert_equal ["row2", "row3"], elem.sort
 
       tsv.add_filter "field:ValueB", ["C"]
 

data/test/rbbt/util/test_log.rb

@@ -0,0 +1,9 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/util/log'
+
+class TestLog < Test::Unit::TestCase
+  def test_color
+    puts Log.color(:green, "green")
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.6.6
+  version: 5.6.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-20 00:00:00.000000000 Z
+date: 2014-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -176,6 +176,7 @@ files:
 - lib/rbbt/tsv/attach/util.rb
 - lib/rbbt/tsv/change_id.rb
 - lib/rbbt/tsv/excel.rb
+- lib/rbbt/tsv/field_index.rb
 - lib/rbbt/tsv/filter.rb
 - lib/rbbt/tsv/index.rb
 - lib/rbbt/tsv/manipulate.rb
@@ -241,6 +242,7 @@ files:
 - share/rbbt_commands/workflow/install
 - share/rbbt_commands/workflow/list
 - share/rbbt_commands/workflow/monitor
+- share/rbbt_commands/workflow/provenance
 - share/rbbt_commands/workflow/remote/add
 - share/rbbt_commands/workflow/remote/list
 - share/rbbt_commands/workflow/remote/remove
@@ -268,6 +270,7 @@ files:
 - test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_manipulate.rb
@@ -280,6 +283,7 @@ files:
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_excel2tsv.rb
 - test/rbbt/util/test_filecache.rb
+- test/rbbt/util/test_log.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_open.rb
 - test/rbbt/util/test_semaphore.rb
@@ -321,6 +325,7 @@ test_files:
 - test/rbbt/util/simpleopt/test_parse.rb
 - test/rbbt/util/test_chain_methods.rb
 - test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/test_log.rb
 - test/rbbt/util/test_open.rb
 - test/rbbt/util/test_R.rb
 - test/rbbt/util/test_colorize.rb
@@ -347,6 +352,7 @@ test_files:
 - test/rbbt/persist/tsv/test_cdb.rb
 - test/rbbt/persist/tsv/test_tokyocabinet.rb
 - test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_change_id.rb
 - test/rbbt/tsv/test_util.rb