rbbt-util 5.13.6 → 5.13.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4cb8cbb3f115088d3b7a4fbe8fb9aa75cf4f4805
-  data.tar.gz: f61c90e38a18f4227f434260c83b6cf2ea48542a
+  metadata.gz: a7fb14a033052fa9f9e469e40ab600e40dfa6f90
+  data.tar.gz: 38a1246aae816a12634ee5681954c11beac3c9af
 SHA512:
-  metadata.gz: 25ac9c560885c7fb66acee8f693a6276f85f74fcaef12bfcc57c3ec59c3562bd3a1a8e25706dce34374c84f7186acaf9c2e901471e0c6002352182a5ecd7534b
-  data.tar.gz: b83ebb5dcdd346679f8a3299fa72ef9dd414590bf5775bcf574639b30be77844dea1bda0a7d6e4b74865992567fae452ed6943fdaaa8be76b503d96d80b37ef9
+  metadata.gz: f55bd2c2fbdd1815df354e7b6bf56c6ef5ec2383acc5ead9fba128ba23048c979bf54a8a1c90392c6982c99e80beebf2a45f5c4f7fef8832aa4a3853292afacc
+  data.tar.gz: f515c4276f796c56826412ffffc23856b221d76be923dc8655234dc1e0cb92781028cc9b0d1e2c92b049724aaaa889b61e4eb79d7e27cca8451fd8749a521051

data/bin/rbbt_Rutil.rb

@@ -1,4 +1,7 @@
 #!/usr/bin/env ruby
+
+ENV["RBBT_LOG"] = "10"
+
 require 'rbbt-util'
 require 'rbbt/util/R'
 

data/lib/rbbt/tsv/attach.rb

@@ -114,7 +114,8 @@ module TSV
         cols2 = parts2.length
       end
 
-      progress_monitor = Progress::Bar.new(size, 0, 100, "Merging fields") if monitor
+      #progress_monitor = Progress::Bar.new(size, 0, 100, "Merging fields") if monitor
+      progress_monitor = Log::ProgressBar.new(size, :desc => "Merging fields") if monitor
 
       entry_hash = options
       entry_hash.delete :sep if entry_hash[:sep] == "\t"

data/lib/rbbt/tsv/manipulate.rb

@@ -162,7 +162,8 @@ module TSV
         desc = @monitor[:desc] if @monitor.include? :desc 
         step = @monitor[:step] if @monitor.include? :step 
       end
-      progress_monitor = Progress::Bar.new(size, 0, step, desc)
+      #progress_monitor = Progress::Bar.new(size, 0, step, desc)
+      progress_monitor = Log::ProgressBar.new(size, :desc => desc)
     else
       progress_monitor = nil
     end

data/lib/rbbt/tsv/parser.rb

@@ -496,7 +496,8 @@ module TSV
           desc = monitor[:desc] if monitor.include? :desc 
           step = monitor[:step] if monitor.include? :step 
         end
-        progress_monitor = Progress::Bar.new(size, 0, step, desc)
+        #progress_monitor = Progress::Bar.new(size, 0, step, desc)
+        progress_monitor = Log::ProgressBar.new(size, :desc => desc)
       else
         progress_monitor = nil
       end

data/lib/rbbt/util/R.rb

@@ -1,5 +1,6 @@
 require 'rbbt/util/cmd'
 require 'rbbt/tsv'
+require 'rbbt/util/R/eval'
 
 module R
 
@@ -80,7 +81,7 @@ module TSV
   def R(script, open_options = {})
     TmpFile.with_file do |f|
       Open.write(f, self.to_s)
-      Log.debug(R.run(
+      io = R.run(
       <<-EOF
 ## Loading tsv into data
 data = rbbt.tsv('#{f}');
@@ -90,12 +91,17 @@ data = rbbt.tsv('#{f}');
 ## Resaving data
 if (! is.null(data)){ rbbt.tsv.write('#{f}', data); }
       EOF
-      ).read)
+      )
+
+      Log.debug(io.read)
+
       open_options = Misc.add_defaults open_options, :type => :list
       if open_options[:raw]
         Open.read(f)
       else
-        TSV.open(f, open_options) unless open_options[:ignore_output]
+        tsv = TSV.open(f, open_options) unless open_options[:ignore_output]
+        tsv.key_field = open_options[:key] if open_options.include? :key
+        tsv
       end
     end
   end

data/lib/rbbt/util/R/eval.rb

@@ -0,0 +1,17 @@
+module R
+  def self.instance
+    @@instance ||= begin
+                     require 'rserve'
+                     @@server_process = CMD.cmd('R CMD Rserve --vanilla', :pipe => true)
+                     Rserve::Connection.new
+                   end
+  end
+
+  def self.eval(cmd)
+    instance.eval(cmd).payload.first
+  end
+
+  def self.eval_a(cmd)
+    instance.eval(cmd).payload
+  end
+end

data/lib/rbbt/util/cmd.rb

@@ -219,15 +219,14 @@ module CMD
         Thread.exit
       end
 
+      ConcurrentStream.setup sout, :pids => [pid], :autojoin => true
       out = StringIO.new sout.read
       sout.close unless sout.closed?
-      #SmartIO.tie out, pid, cmd, post, in_content, sin, serr
 
       Process.waitpid pid
 
       if not $?.success?
-        exception      = ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
-        raise exception
+        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
       else
         Log.log err, stderr if Integer === stderr and log
       end

data/lib/rbbt/util/concurrency/processes.rb

@@ -2,7 +2,6 @@ require 'rbbt/util/concurrency/processes/worker'
 require 'rbbt/util/concurrency/processes/socket'
 
 class RbbtProcessQueue
-  #{{{ RbbtProcessQueue
 
   attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup, :join
   def initialize(num_processes, cleanup = nil, join = nil)

data/lib/rbbt/util/log/progress/report.rb

@@ -34,7 +34,7 @@ module Log
 
       @mean_max ||= 0
       if @history.length > 3
-        mean = Misc.mean(@history)
+        mean = @mean = Misc.mean(@history)
         @mean_max = mean if mean > @mean_max
       end
 
@@ -60,7 +60,7 @@ module Log
 
       used = time - @start
       if @mean_max and @mean_max > 0
-        eta =  (@max - @ticks) / @mean_max
+        eta =  (@max - @ticks) / @mean
       else
         eta =  (@max - @ticks) / (@ticks/used)
       end
@@ -98,7 +98,9 @@ module Log
 
     def done(io = STDERR)
       done_msg = Log.color(:magenta, desc) << " " << Log.color(:green, "done")
-      done_msg << " " << Log.color(:yellow, (@ticks).to_s) << " in " << Log.color(:blue, (Time.now - @start).to_i.to_s) << " sec." 
+      ellapsed = (Time.now - @start).to_i
+      ellapsed = [ellapsed/3600, ellapsed/60 % 60, ellapsed % 60].map{|t| "%02i" % t }.join(':')
+      done_msg << " " << Log.color(:blue, (@ticks).to_s) << " in " << Log.color(:green, ellapsed)
       @last_count = 0
       @last_time = @start
       done_msg << " (" << thr_msg << ")"

data/lib/rbbt/util/misc/pipes.rb

@@ -221,11 +221,11 @@ module Misc
         rescue Aborted
           Log.warn "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
           content.abort if content.respond_to? :abort
-          Open.rm_f path if File.exists? path
+          Open.rm path if File.exists? path
         rescue Exception
           Log.warn "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
           content.abort if content.respond_to? :abort
-          Open.rm_f path if File.exists? path
+          Open.rm path if File.exists? path
           raise $!
         ensure
           FileUtils.rm_f tmp_path if File.exists? tmp_path
@@ -244,7 +244,7 @@ module Misc
     end
   end
 
-  def self.sort_stream(stream, header_hash = "#")
+  def self.sort_stream(stream, header_hash = "#", cmd_args = nil)
     Misc.open_pipe do |sin|
       begin
         if defined? Step and Step === stream
@@ -271,7 +271,7 @@ module Misc
           end
         end
 
-        sorted = CMD.cmd("sort", :in => line_stream, :pipe => true)
+        sorted = CMD.cmd("sort #{cmd_args || ""}", :in => line_stream, :pipe => true)
 
         while block = sorted.read(2048)
           sin.write block

data/lib/rbbt/workflow/step/run.rb

@@ -27,12 +27,10 @@ class Step
 
         else
           Log.medium "Duplicating stream #{ Misc.fingerprint(stream) }"
-          new = Misc.dup_stream(current)
-          Log.medium str << Log.color(:green, " -> ") << "#{new.inspect}"
-          new
+          Misc.dup_stream(current)
         end
       end
-    when TSV::Dumper, TSV::Parser
+    when TSV::Dumper#, TSV::Parser
       stream = stream.stream
       return stream if stream.closed?
 
@@ -90,6 +88,7 @@ class Step
   end
 
   def exec(no_load=false)
+    dup_inputs
     dependencies.each{|dependency| dependency.exec(no_load) }
     @mutex.synchronize do
       @result = self._exec

data/share/rbbt_commands/tsv/sort

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Sort a TSV file, mantaining headers
+
+$ rbbt tsv sort file.tsv
+
+Display summary information. Works with Tokyocabinet HDB and BDB as well.
+
+-hh--header_hash* Change the character used to mark the header line (defaults to #)
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-' or file.nil?
+
+rest = ARGV.inject(""){|acc,e| acc << " '#{e}'" }
+
+file_io = TSV.get_stream(file)
+stream = Misc.sort_stream(file_io, options[:header_hash] || "#", rest)
+
+begin
+  while line = stream.gets
+    puts line
+  end
+rescue Exception
+  exit -1
+end

data/share/rbbt_commands/workflow/example

@@ -67,9 +67,9 @@ workflow = Workflow.require_workflow workflow
 tasks = task ? [task] : workflow.libdir.examples.glob('*').collect{|file| File.basename file }
 
 task_result = {}
-TSV.traverse tasks, :threads => 5 do |task|
+TSV.traverse tasks do |task|
   names = name ? [name] : workflow.libdir.examples[task].glob('*').collect{|file| File.basename file }
-  TSV.traverse names, :threads => 5 do |name|
+  TSV.traverse names  do |name|
     success = run_task workflow, task, name
     task_result[[task, name]] = success
   end

data/test/rbbt/util/R/test_eval.rb

@@ -0,0 +1,16 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/R/eval'
+
+class TestREval < Test::Unit::TestCase
+  def test_get
+    Misc.benchmark(100) do
+      a = R.eval <<-EOF
+    p = 12
+    a = p * 2
+    c(a)
+      EOF
+      assert_equal 12 * 2, a
+    end
+  end
+end
+

data/test/rbbt/util/misc/test_pipes.rb

@@ -5,7 +5,7 @@ require 'rbbt/util/misc'
 
 class TestMiscPipes < Test::Unit::TestCase
 
-  def test_collapse_stream
+  def _test_collapse_stream
     text=<<-EOF
 row1 A B C
 row1 a b c
@@ -19,7 +19,7 @@ row2 aa bb cc
     assert_equal ["BB", "bb"], tsv["row2"][1]
   end
 
-  def test_paste_stream
+  def _test_paste_stream
     text1=<<-EOF
 row1 A B C
 row2 AA BB CC
@@ -46,7 +46,7 @@ row3 ccc
     assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
   end
 
-  def test_sort_stream
+  def _test_sort_stream
     text =<<-EOF
 #: :sep=" "
 #Row LabelA LabelB LabelC
@@ -69,10 +69,19 @@ row3 AAA BBB CCC
 row1 A B C
     EOF
 
+    text = text * 10000
     TmpFile.with_file(text) do |tmp|
       io = Open.open(tmp)
       dup = Misc.dup_stream(io)
+      Misc.consume_stream io, true
       assert_equal text, dup.read
+    end
+
+
+    TmpFile.with_file(text) do |tmp|
+      io = Open.open(tmp)
+      dup = Misc.dup_stream(io)
+      Misc.consume_stream dup, true
       assert_equal text, io.read
     end
   end

data/test/rbbt/util/test_R.rb

@@ -18,5 +18,13 @@ data = data + 1
     tsv = TSV.setup({"a" => [1], "b" => [2]}, :type => :list, :key_field => "Letter", :fields => ["Number"])
     assert tsv.transpose("Field").include? "Number"
   end
+
+  def test_error
+    assert_raise ProcessFailed do
+      R.run <<-EOF
+        fadsfasdf
+      EOF
+    end
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.6
+  version: 5.13.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-07 00:00:00.000000000 Z
+date: 2014-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -161,6 +161,7 @@ files:
 - lib/rbbt/tsv/stream.rb
 - lib/rbbt/tsv/util.rb
 - lib/rbbt/util/R.rb
+- lib/rbbt/util/R/eval.rb
 - lib/rbbt/util/chain_methods.rb
 - lib/rbbt/util/cmd.rb
 - lib/rbbt/util/color.rb
@@ -250,6 +251,7 @@ files:
 - share/rbbt_commands/tsv/info
 - share/rbbt_commands/tsv/json
 - share/rbbt_commands/tsv/slice
+- share/rbbt_commands/tsv/sort
 - share/rbbt_commands/tsv/subset
 - share/rbbt_commands/tsv/unzip
 - share/rbbt_commands/tsv/values
@@ -299,6 +301,7 @@ files:
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_stream.rb
 - test/rbbt/tsv/test_util.rb
+- test/rbbt/util/R/test_eval.rb
 - test/rbbt/util/concurrency/processes/test_socket.rb
 - test/rbbt/util/concurrency/test_processes.rb
 - test/rbbt/util/concurrency/test_threads.rb
@@ -374,6 +377,7 @@ test_files:
 - test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_tmpfile.rb
+- test/rbbt/util/R/test_eval.rb
 - test/rbbt/test_association.rb
 - test/rbbt/test_resource.rb
 - test/rbbt/test_entity.rb