rbbt-util 5.19.34 → 5.19.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 891d8471ba86780e53a5363adac579490a665f05
-  data.tar.gz: f80a9947a157c9aaa5b8de42f2119004afe391b1
+  metadata.gz: 71c4c5187485430e7ea4977a2a46002f2a5caa81
+  data.tar.gz: 60e13741b32e65b17e63621a0dcdd021b467b2ff
 SHA512:
-  metadata.gz: 330ec02481ecae70a0748a0d33f6478576b185a8db7aac15624a2df5473f4762263c320c244f30e80977e377c1daaf82943cb387930e34ee10807107d8a4fbbd
-  data.tar.gz: e49d24357ed693a1cedfe88ae3b9c43d92f28080de7cacf627bc6c5aa2bd3d0c072c07c74a30b9a10f7e7c7f6a46b88771f1e3e40d3967cbe5ce2a25f3c7938a
+  metadata.gz: 70760328e15502ce16d06483def0e5d6c7ee75d20d33e7891666e2a10c8cebb275a5a277be0afd0810fdecfa99f3d27ee48ddf93335b52caa94f629102fac250
+  data.tar.gz: 7310fddb61cc783289304aabd83b77bf47b2666d1f31ba90a7d15c91b701016b3272d2d2a7503b8b3697fd62a56278e420be3caf81efdfcf3f18c6bf629e138f

data/etc/app.d/requires.rb

@@ -2,6 +2,10 @@
 Rbbt.etc.requires.read.split("\n").each do |file|
   next if file.empty?
   Log.debug("requiring #{ file }")
-  require file
+  begin
+    require file
+  rescue Exception
+    Log.warn "Could not require #{ file }"
+  end
 end if Rbbt.etc.requires.exists?
 

data/lib/rbbt/rest/client/step.rb

@@ -1,3 +1,4 @@
+require 'excon'
 class WorkflowRESTClient
   class RemoteStep < Step
 
@@ -89,18 +90,25 @@ class WorkflowRESTClient
       @mutex.synchronize do
         @result ||= begin
                       if @is_exec
-                        exec_job 
+                        exec(noload)
+                      elsif noload == :stream
+                        _run_job(:stream)
                       else
                         init_job 
                         self.load
                       end
                     end
       end
+      return @result if noload == :stream
       noload ? path + '?_format=raw' : @result
     end
 
-    def exec(*args)
-      exec_job
+    def exec(noload = false)
+      if noload == :stream
+        _run_job(:exec)
+      else
+        exec_job 
+      end
     end
 
     def join
@@ -124,18 +132,28 @@ class WorkflowRESTClient
     end
 
     def load_res(res, result_type = nil)
-      join
+      stream = true if res.respond_to? :read
+      join unless stream
       result_type ||= self.result_type
       case result_type
       when :string
-        res
+        stream ? res.read : res
       when :boolean
-        res == "true"
+        (stream ? res.read : res) == 'true'
       when :tsv
-        TSV.open(StringIO.new(res))
+        if stream
+          TSV.open(res, :monitor => true)
+        else
+          TSV.open(StringIO.new(res))
+        end
       when :annotations
-        Annotated.load_tsv(TSV.open(StringIO.new(res)))
+        if stream
+          Annotated.load_tsv(TSV.open(res))
+        else
+          Annotated.load_tsv(TSV.open(StringIO.new(res)))
+        end
       when :array
+        (stream ? res.read : res).split("\n")
         res.split("\n")
       else
         JSON.parse res
@@ -147,10 +165,33 @@ class WorkflowRESTClient
       load_res get
     end
     
-    def exec_job
-      res = WorkflowRESTClient.capture_exception do
-        RestClient.post(URI.encode(File.join(base_url, task.to_s)), inputs.merge(:_cache_type => :exec, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json))
+    def _run_job(cache_type = :async)
+      WorkflowRESTClient.capture_exception do
+        url = URI.encode(File.join(base_url, task.to_s))
+        task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
+
+        sout, sin = Misc.pipe
+        streamer = lambda do |c|
+          sin.write c
+        end
+
+        Thread.new do
+          bl = lambda do |rok|
+            rok.read_body do |c,_a, _b|
+              sin.write c
+            end
+            sin.close
+          end
+
+          RestClient::Request.execute(:method => :post, :url => url, :payload => task_params, :block_response => bl)
+        end
+
+        Zlib::GzipReader.new(sout)
       end
+    end
+
+    def exec_job
+      res = _run_job(:exec)
       load_res res, result_type == :array ? :json : result_type
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -267,6 +267,8 @@ module TSV
         end
       when Array
         traverse_array(obj, options, &block)
+      when Set
+        traverse_array(obj.to_a, options, &block)
       when String
         if Open.remote? obj or Misc.is_filename? obj
           Open.open(obj) do |s|

data/lib/rbbt/tsv/parser.rb

@@ -554,6 +554,15 @@ module TSV
 
         progress_monitor.bytes = true
         progress_monitor.max =  size unless size.to_i == 0
+      elsif monitor
+        desc = "Parsing Stream"
+        step = 100
+        size = nil
+        if Hash === monitor
+          desc = monitor[:desc] if monitor.include? :desc 
+          step = monitor[:step] if monitor.include? :step 
+        end
+        progress_monitor = Log::ProgressBar.new_bar(size, :desc => desc, :bytes => true)
       end
 
       # parser 

data/lib/rbbt/tsv/stream.rb

@@ -1,18 +1,21 @@
 require 'rbbt/tsv/dumper'
 module TSV
 
-  def self.collapse_stream(input, options = {})
-    options = Misc.add_defaults options, :sep => "\t"
+  def self.collapse_stream(input, options = {}, &block)
+    options = Misc.add_defaults options, :sep => "\t", :header_hash => '#', :uniq => true
     input_stream = TSV.get_stream input
 
-    sorted_input_stream = Misc.sort_stream input_stream
+    header_hash = options[:header_hash]
+    cmd_args = options[:uniq] ? "-u" : nil
+
+    sorted_input_stream = Misc.sort_stream input_stream, header_hash, cmd_args
 
     parser = TSV::Parser.new sorted_input_stream, options.dup
     dumper = TSV::Dumper.new parser
     header = TSV.header_lines(parser.key_field, parser.fields, parser.options)
     dumper.close_in
     dumper.close_out
-    dumper.stream = Misc.collapse_stream parser.stream, parser.first_line, parser.sep, header
+    dumper.stream = Misc.collapse_stream parser.stream, parser.first_line, parser.sep, header, &block
     dumper
   end
  
@@ -65,7 +68,7 @@ module TSV
 
         if fix_flat and parser.type == :flat and parser.first_line
           parts = lines[-1].split("\t")
-          lines[-1] = [parts[0], parts[1..-1]*"|"] * "\t"
+          lines[-1] = [parts[0], (parts[1..-1] || [])*"|"] * "\t"
           TSV.stream_flat2double(parser.stream).stream
         else
           parser.stream
@@ -220,6 +223,7 @@ module TSV
     parser = TSV::Parser.new TSV.get_stream(stream), :type => :flat
     dumper_options = parser.options.merge(options).merge(:type => :double)
     dumper = TSV::Dumper.new dumper_options
+    dumper.init
     TSV.traverse parser, :into => dumper do |key,values|
       key = key.first if Array === key
       values = [values] unless Array === values

data/lib/rbbt/util/docker.rb

@@ -20,7 +20,11 @@ module Docker
         when File 
           FileUtils.cp obj.filename, directory[name]
         when IO
-          Open.write(tmpfile[name], obj)
+          begin
+            Open.write(directory[name], obj)
+          ensure
+            obj.join if obj.respond_to?(:join) and not obj.joined?
+          end
         when String
           if obj.length < 256 and File.exists?(obj)
             FileUtils.cp obj, directory[name]
@@ -39,7 +43,11 @@ module Docker
           when File 
             FileUtils.cp obj.filename, tmpfile[name]
           when IO
-            Open.write(tmpfile[name], obj)
+            begin
+              Open.write(tmpfile[name], obj)
+            ensure
+              obj.join if obj.respond_to?(:join) and not obj.joined?
+            end
           when String
             if obj.length < 256 and File.exists?(obj)
               FileUtils.cp obj, tmpfile[name]

data/lib/rbbt/util/log/progress.rb

@@ -16,7 +16,7 @@ module Log
       @last_count = nil
       @last_percent = nil
       @depth = depth
-      @desc = desc.nil? ? nil : desc.gsub(/\n/,' ')
+      @desc = desc.nil? ? "Progress" : desc.gsub(/\n/,' ')
       @file = file
     end
 

data/lib/rbbt/util/misc/pipes.rb

@@ -159,7 +159,7 @@ module Misc
     str
   end
 
-  def self.consume_stream(io, in_thread = false, into = nil)
+  def self.consume_stream(io, in_thread = false, into = nil, into_close = true)
     return if Path === io
     return unless io.respond_to? :read 
     if io.respond_to? :closed? and io.closed?
@@ -169,10 +169,15 @@ module Misc
 
     if in_thread
       Thread.new do
-        consume_stream(io, false)
+        consume_stream(io, false, into, into_close)
       end
     else
-      Log.medium "Consuming stream #{Misc.fingerprint io}"
+      if into
+        Log.medium "Consuming stream #{Misc.fingerprint io} -> #{Misc.fingerprint into}"
+      else
+        Log.medium "Consuming stream #{Misc.fingerprint io}"
+      end
+
       begin
         into = into.find if Path === into
         into = Open.open(into, :mode => 'w') if String === into 
@@ -182,6 +187,8 @@ module Misc
         end
         io.join if io.respond_to? :join
         io.close unless io.closed?
+        into.close if into and into_close and not into.closed?
+        into.join if into and into_close and into.respond_to?(:joined?) and not into.joined?
       rescue Aborted
         Log.medium "Consume stream aborted #{Misc.fingerprint io}"
         io.abort if io.respond_to? :abort
@@ -344,7 +351,7 @@ module Misc
     end
   end
 
-  def self.collapse_stream(s, line = nil, sep = "\t", header = nil)
+  def self.collapse_stream(s, line = nil, sep = "\t", header = nil, &block)
     sep ||= "\t"
     Misc.open_pipe do |sin|
       sin.puts header if header
@@ -371,15 +378,28 @@ module Misc
             (parts.length..current_parts.length-1).to_a.each do |pos|
               current_parts[pos] = current_parts[pos] << "|" << ""
             end
+          when current_key.nil?
+            current_key = key
+            current_parts = parts
           when current_key != key
-            sin.puts [current_key, current_parts].flatten * sep
+            if block_given?
+              res = block.call(current_parts)
+              sin.puts [current_key, res] * sep
+            else
+              sin.puts [current_key, current_parts].flatten * sep
+            end 
             current_key = key
             current_parts = parts
           end
           line = s.gets
         end
 
-        sin.puts [current_key, current_parts].flatten * sep unless current_key.nil?
+        if block_given?
+          res = block.call(current_parts)
+          sin.puts [current_key, res] * sep
+        else
+          sin.puts [current_key, current_parts].flatten * sep
+        end unless current_key.nil?
       end
     end
   end

data/lib/rbbt/workflow/accessor.rb

@@ -242,7 +242,13 @@ class Step
     Step.log_progress(status, options, file(:progress), &block)
   end
 
-  def progress_bar(msg = "Progress", options = {})
+  def progress_bar(msg = "Progress", options = nil)
+    if Hash === msg and options.nil?
+      options = msg
+      msg = nil
+    end
+    options = {} if options.nil?
+
     max = options[:max]
     Log::ProgressBar.new_bar(max, {:desc => msg, :file => file(:progress)}.merge(options))
   end
@@ -565,7 +571,14 @@ module Workflow
                 #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
                 _inputs[i] = rec_dependency
               else
-                _inputs[i] = rec_dependency.run
+                rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
+                rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
+                if rec_dependency.streaming? and rec_dependency.running?
+                  _inputs[i] = rec_dependency.join.load
+                else
+                  rec_dependency.run(true).join
+                  _inputs[i] = rec_dependency.load
+                end
               end
             end
           else

data/share/Rlib/util.R

@@ -18,8 +18,8 @@ rbbt.ruby <- function(code, load = TRUE, flat = FALSE, type = 'tsv', ...){
           return(data)
       }
 
-      if(type == 'list'){
-          data = scan(file, ...)
+      if(type == 'list' || type == 'array'){
+          data = scan(file, what='string', sep="\n", ...)
           return(data);
       }
 
@@ -33,6 +33,30 @@ rbbt.ruby <- function(code, load = TRUE, flat = FALSE, type = 'tsv', ...){
   }
 }
 
+rbbt.job <- function(workflow, task, load=TRUE, flat = FALSE, type = 'tsv', jobname="R.Default", code='', ...){
+
+    str = "require 'rbbt/workflow'"
+
+    str = paste(str, code, sep="\n")
+
+    str = paste(str, paste("Workflow.require_workflow '", workflow, "'", sep=""), sep="\n")
+
+    args_list = list(...)
+    args_strs = c()
+    for (input in names(args_list)){
+        value = args_list[[input]]
+        if (!is.numeric(value)){
+            value = paste("'", value, "'", sep="")
+        }
+        args_strs = c(args_strs, paste(":",input,' => ',value, sep=""))
+    }
+
+    args_str = paste(args_strs, sep=",")
+    str = paste(str, paste(workflow, '.job(:', task, ", '", jobname, "', ", args_str,').produce.path', sep=""), sep="\n")
+    cat(str)
+    rbbt.ruby(str, load, flat, type)
+}
+
 rbbt.ruby.substitutions <- function(script, substitutions = list(), ...){
     
     for (keyword in names(substitutions)){

data/share/rbbt_commands/study/task

@@ -203,7 +203,7 @@ if do_fork
   end
   raise job.messages.last if job.error?
 else
-  res = job.run(true)
+  res = job.run(:stream)
 end
 
 if options.delete(:printname)
@@ -216,7 +216,13 @@ end
 if Step === res
   puts Open.read(res.path) if File.exists? res.path
 else
-  puts res
+  if res.respond_to? :gets
+    while line = res.gets
+      puts line
+    end
+  else
+    puts res
+  end
 end
 
 exit 0

data/share/rbbt_commands/workflow/server

@@ -67,10 +67,6 @@ app.use Rack::Deflater
 
 load Rbbt.etc['app.d/resources.rb'].find
 
-load Rbbt.etc['app.d/requires.rb'].find
-
-load Rbbt.etc['app.d/entities.rb'].find
-
 app.class_eval do
   eval Rbbt.etc['app.d/finder.rb'].read
 end
@@ -83,6 +79,10 @@ if options[:workflows]
   end
 end
 
+load Rbbt.etc['app.d/requires.rb'].find
+
+load Rbbt.etc['app.d/entities.rb'].find
+
 
 if options[:views] and not options[:views].empty?
   Sinatra::RbbtRESTMain.add_resource_path(Path.setup(options[:views]), true)

data/share/rbbt_commands/workflow/task

@@ -302,8 +302,12 @@ begin
   end
 
   if do_exec or (job.respond_to?(:is_exec) and job.is_exec)
-    res = job.exec(true)
+    res = job.exec(:stream)
     case
+    when res.respond_to?(:gets)
+      while block = res.read(2048)
+        out.write block
+      end
     when Array === res
       out.puts res * "\n"
     when TSV === res
@@ -331,7 +335,7 @@ begin
 
     job.fork
   else
-    job.run(true)
+    job.run(:stream)
     res = job
   end
 
@@ -427,9 +431,23 @@ end
 
 case res
 when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
-  out.puts res.load
+  res = job.result
+  if res.respond_to? :gets
+    while block = res.read(2048) do
+      out.write block
+    end #unless io.closed? 
+    res.join if res.respond_to? :join
+  else
+    puts res.to_s
+  end
 when Step
-  if IO === res.result
+  if step.streaming?
+    io = res.get_stream
+    while block = io.read(2048) do
+      out.write block
+    end #unless io.closed? 
+    io.join if io.respond_to? :join
+  elsif IO === res.result
     begin
       io = res.get_stream
       while block = io.read(2048) do

data/test/rbbt/tsv/test_stream.rb

@@ -193,7 +193,6 @@ row2 AAA
     s1 = StringIO.new text1
     s2 = StringIO.new text2
     tsv = TSV.open TSV.paste_streams([s1,s2], :sep => " ", :type => :double, :sort => false, :same_fields => true)
-    ppp tsv.to_s
     assert_equal "Row", tsv.key_field
     assert_equal ["AA", "AAA"], tsv["row2"][0]
   end

data/test/rbbt/util/misc/test_pipes.rb

@@ -19,6 +19,23 @@ row2 aa bb cc
     assert_equal ["BB", "bb"], tsv["row2"][1]
   end
 
+  def test_collapse_sum
+    text=<<-EOF
+row1 12
+row1 4
+row2 10
+row2 6
+    EOF
+
+    s = StringIO.new text
+    stream = Misc.collapse_stream(s,nil, " ") do |parts|
+      next nil if parts.empty?
+      parts.first.split("|").collect{|p| p.to_f}.inject(0){|acc,e| acc += e}.to_s
+    end
+    tsv = TSV.open  stream, :sep => " " 
+    ppp tsv.to_s
+  end
+
   def test_collapse_stream_gap
     text=<<-EOF
 row2 AA BB 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.19.34
+  version: 5.19.35
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-20 00:00:00.000000000 Z
+date: 2016-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake