rbbt-util 5.7.0 → 5.8.0

data/lib/rbbt/util/semaphore.rb

@@ -50,6 +50,15 @@ void post_semaphore(char* name){
     EOF
   end
 
+  def self.synchronize(sem)
+    RbbtSemaphore.wait_semaphore(sem)
+    begin
+      yield
+    ensure
+      RbbtSemaphore.post_semaphore(sem)
+    end
+  end
+
   def self.with_semaphore(size, file = nil)
     file = Misc.digest(rand.to_s) if file.nil?
     file.gsub!('/', '_')
@@ -68,11 +77,8 @@ void post_semaphore(char* name){
         pids = elems.collect do |elem| 
           Process.fork do 
             begin
-              RbbtSemaphore.wait_semaphore(file)
-              begin
+              RbbtSemaphore.synchronize(file) do
                 yield elem
-              ensure
-                RbbtSemaphore.post_semaphore(file)
               end
             rescue Interrupt
               Log.error "Process #{Process.pid} was aborted"

data/lib/rbbt/util/simpleopt/accessor.rb

@@ -47,4 +47,9 @@ module SOPT
       self.input_descriptions.delete input
     end
   end
+
+  def self.usage
+    puts SOPT.doc
+    exit 0
+  end
 end

data/lib/rbbt/util/simpleopt/doc.rb

@@ -73,10 +73,8 @@ module SOPT
 
   def self.doc
     doc = <<-EOF
-#{Log.color :magenta}
-#{command}(1) -- #{summary}
-#{"=" * (command.length + summary.length + 7)}
-#{Log.color :reset}
+#{Log.color :magenta}#{command}(1) -- #{summary}
+#{"=" * (command.length + summary.length + 7)}#{Log.color :reset}
 
 #{ Log.color :magenta, "## SYNOPSYS"}
 

data/lib/rbbt/workflow/accessor.rb

@@ -94,16 +94,27 @@ class Step
     set_info(:messages, (messages || []) << message)
   end
 
-  def log(status, message = nil, do_log = true)
+  attr_accessor :last_log
+  def last_log
+    @last_log ||= Time.now
+  end
 
+  def log(status, message = nil)
     if message
-      Log.medium "[#{ status }] #{ message }: #{path}"
-    else
-      Log.medium "[#{ status }]: #{path}"
-    end if do_log
-
-    self.status = status
-    message(message) unless message.nil?
+      Log.medium do 
+        now = Time.now
+        str = "+#{(now - last_log).to_i} #{ Log.color :cyan, status.to_s }: #{ message } -- #{path}"
+        @last_log = now
+        str
+      end
+    else                                                                      
+      Log.medium do
+        now = Time.now
+        str = "+#{(now - last_log).to_i} #{ Log.color :cyan, status.to_s } -- #{path}"
+        @last_log = now
+        str
+      end
+    end
   end
 
   def started?
@@ -213,11 +224,27 @@ class Step
 end
 
 module Workflow
+
+  attr_accessor :last_log
+  def last_log
+    @@last_log ||= Time.now
+  end
+
   def log(status, message = nil)
     if message
-      Log.low "#{ status }: #{ message }"
-    else
-      Log.low "#{ status }"
+      Log.medium do 
+        now = Time.now
+        str = "+#{(now - last_log).to_i} #{ Log.color :cyan, status.to_s }: #{ message }"
+        @@last_log = now
+        str
+      end
+    else                                                                      
+      Log.medium do
+        now = Time.now
+        str = "+#{(now - last_log).to_i} #{ Log.color :cyan, status.to_s }"
+        @@last_log = now
+        str
+      end
     end
   end
 
@@ -315,7 +342,7 @@ module Workflow
       if inputs.any? or dependencies.any?
         tagged_jobname = case TAG
                          when :hash
-                           jobname + '_' + Misc.digest((inputs * "\n" + ";" + dependencies.collect{|dep| dep.name} * "\n"))
+                           jobname + '_' + Misc.digest((inputs.collect{|i| Misc.fingerprint(i)} * "," + ";" + dependencies.collect{|dep| dep.name } * "\n"))
                          else
                            jobname
                          end

data/lib/rbbt/workflow/step.rb

@@ -10,8 +10,6 @@ class Step
   attr_accessor :pid
   attr_accessor :exec
 
-  class Aborted < Exception; end
-
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
     @path = path
@@ -123,7 +121,7 @@ class Step
         end
       }
       
-      Log.medium("Starting task #{task.name || ""} [#{Process.pid}]: #{ path }")
+      Log.medium{"#{Log.color :magenta, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]: #{ path }"}
       set_info :status, :started
 
       set_info :started, (start_time = Time.now)
@@ -132,7 +130,7 @@ class Step
 
       res = begin
               exec
-            rescue Step::Aborted
+            rescue Aborted
               log(:error, "Aborted")
 
               children_pids = info[:children_pids]
@@ -165,7 +163,7 @@ class Step
       set_info :status, :done
       set_info :done, (done_time = Time.now)
       set_info :time_elapsed, done_time - start_time
-      Log.medium("Completed task #{task.name || ""} [#{Process.pid}]: #{ path }")
+      Log.medium{"#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]: #{ path }"}
 
       res
     end
@@ -180,13 +178,13 @@ class Step
   def fork(semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil?
     @pid = Process.fork do
-      trap(:INT) { raise Step::Aborted.new "INT signal recieved" }
+      trap(:INT) { raise Aborted.new "INT signal recieved" }
       begin
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
         FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
         begin
           run(true)
-        rescue Step::Aborted
+        rescue Aborted
           Log.debug{"Forked process aborted: #{path}"}
           log :aborted, "Aborted"
           raise $!

data/share/rbbt_commands/benchmark/pthrough

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+
+file, *sizes = ARGV
+
+sizes.each do |size|
+  size = size.to_i
+
+  %w(--parallel=cpu --parallel=thr --noparallel).each do |parallel|
+    Log.info Log.color(:red, parallel) + " " + Log.color(:blue, size.to_s)
+
+    start = Time.now
+    `head -n #{ size } "#{file}" | drbbt task Structure mutated_isoform_neighbour_annotation -g - -cl --log 10 -pn #{parallel}`
+    Log.info Log.color(:red, parallel) + " " + Log.color(:blue, size.to_s) + " " + Log.color(:yellow, (Time.now - start).to_s)
+  end
+end
+

data/share/rbbt_commands/color

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Matches text and colors it.
+
+$ rbbt color <file|-> (<color> <pattern>)+
+
+Use - to read from STDIN. Example: `rbbt color - red "/.*rbbt.*/"`
+
+-h--help Print help
+
+EOF
+rbbt_usage and exit 0 if options[:help]
+
+file, *rest = ARGV
+
+if file == '-'
+  text = STDIN.read
+else
+  text = Open.read(file)
+end
+
+while rest.any?
+  color = rest.shift
+  pattern = rest.shift
+  color = color.to_sym
+  if pattern =~ /^\/(.*)\/$/
+    text.gsub!(Regexp.compile($1)){|match| Log.color color, match }
+  else
+    text.gsub!(pattern){|match| Log.color color, match }
+  end
+end
+
+puts text
+
+

data/share/rbbt_commands/stat/density

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+require 'rbbt'
+require 'rbbt/util/simpleopt'
+
+require 'rbbt/util/R'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Calculate density 
+
+$ rbbt stat density <file>
+
+Display summary information. Works with Tokyocabinet HDB and DBD as well.
+
+-tch--tokyocabinet File is a TC HDB
+-tcb--tokyocabinet_bd File is a TC BDB
+-t--type* Type of tsv (single, list, double, flat)
+-h--header_hash* Change the character used to mark the header line (defaults to #)
+-k--key_field* Change the key field
+-f--fields* Change the fields to load
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file = ARGV.shift
+
+if file == '-' or file.nil?
+  file = STDIN 
+else
+  file = Open.open file
+end
+
+
+values = file.read.split("\n").collect{|v| v.to_f}
+
+res = TmpFile.with_file do |tmp|
+  R.run <<-EOF
+values = #{R.ruby2R values}
+d = density(values)
+df = data.frame(x=d$x, y=d$y)
+rbbt.tsv.write(file='#{ tmp }', df)
+  EOF
+
+  Open.read(tmp)
+end
+
+puts res.split("\n")[1..-1].collect{|l| l.split("\t")[1,2] * "\t"} * "\n"

data/share/rbbt_commands/tsv/info

@@ -3,10 +3,25 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 
-options = SOPT.get("-tch--tokyocabinet:-tcb--tokyocabinet_bd:-t--type*:-h--header_hash*:-k--key_field*:-f--fields*")
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
-options[:fields] = options[:fields].split(/,\|/) if options[:fields]
-options[:header_hash] = options["header_hash"]
+options = SOPT.setup <<EOF
+Inspect a TSV file 
+
+$ rbbt tsv info [options] file.tsv
+
+Display summary information. Works with Tokyocabinet HDB and DBD as well.
+
+-tch--tokyocabinet File is a TC HDB
+-tcb--tokyocabinet_bd File is a TC BDB
+-t--type* Type of tsv (single, list, double, flat)
+-h--header_hash* Change the character used to mark the header line (defaults to #)
+-k--key_field* Change the key field
+-f--fields* Change the fields to load
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
 
 file = ARGV.shift
 
@@ -14,6 +29,9 @@ file = STDIN if file == '-'
 
 raise ParameterException, "Please specify the tsv file as argument" if file.nil?
 
+options[:fields] = options[:fields].split(/,\|/) if options[:fields]
+options[:header_hash] = options["header_hash"]
+
 case
 when options[:tokyocabinet]
   tsv = Persist.open_tokyocabinet(file, false)

data/share/rbbt_commands/tsv/slice

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Slice column from tsv
+
+$ rbbt tsv slice [options] file.tsv -f "Field 1"
+
+Display summary informations. Works with Tokyocabinet HDB and DBD as well.
+
+-tch--tokyocabinet File is a TC HDB
+-tcb--tokyocabinet_bd File is a TC BDB
+-hh--header_hash* Change the character used to mark the header line (defaults to #)
+-f--fields* Field to slice (comma-separated)
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+raise ParameterException, "Please specify the tsv file as argument" if file.nil?
+
+fields = options[:fields]
+raise ParameterException, "Please specify the fields to slice" if fields.nil?
+
+options[:header_hash] = options["header_hash"]
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+  puts tsv.summary
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+  puts tsv.summary
+else
+  tsv = TSV.open(file, options)
+end
+
+puts tsv.slice(fields.split(","))

data/share/rbbt_commands/tsv/subset

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Subset entries in a tsv
+
+$ rbbt tsv subset [options] file.tsv --subset key1,key2,key3
+
+Subsets entries from a TSV file from a given list. Works with Tokyocabinet HDB and DBD as well.
+
+-tch--tokyocabinet File is a TC HDB
+-tcb--tokyocabinet_bd File is a TC BDB
+-hh--header_hash* Change the character used to mark the header line (defaults to #)
+-s--subset* Subset of samples (Comma-separated of file)
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+raise ParameterException, "Please specify the tsv file as argument" if file.nil?
+
+subset = options[:subset]
+raise ParameterException, "Please specify a subset of keys" if subset.nil?
+
+if File.exists?(subset)
+  subset = Open.read(subset).split("\n")
+else
+  subset = subset.split(',')
+end
+
+options[:fields] = options[:fields].split(/,\|/) if options[:fields]
+options[:header_hash] = options["header_hash"]
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+  puts tsv.summary
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+  puts tsv.summary
+else
+  tsv = TSV.open(file, options)
+end
+
+puts tsv.select(subset)

data/share/rbbt_commands/tsv/values

@@ -15,6 +15,7 @@ Use - to read from STDIN
 -tcb--tokyocabinet_bd File is a tokyocabinet B database
 -f--field* Limit to a particular field
 -h--help Print this help
+-l--lines Separate in lines
 
 EOF
 rbbt_usage and exit 0 if options[:help]
@@ -37,6 +38,11 @@ else
   tsv = TSV.open(file)
 end
 
+field = options[:field]
 tsv = tsv.slice(field) if field
 
-tsv.each{|k,v| puts (Array === v ? v.flatten*"\t" : v.to_s ) }
+if options[:lines]
+  tsv.each{|k,v| puts (Array === v ? v.flatten*"\n" : v.to_s ) }
+else
+  tsv.each{|k,v| puts (Array === v ? v.flatten*"\t" : v.to_s ) }
+end