rbbt-util 5.9.12 → 5.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 611734c0287b691572f51267904bec6af278eba1
-  data.tar.gz: c718a679f44bebb62b3f235e71765f9deebbb13d
+  metadata.gz: f5de94d0ec4441889212dc37780f880b38809d71
+  data.tar.gz: 44acad91d91a0a8f7c351d32cd2dba45ca9fc058
 SHA512:
-  metadata.gz: 2bb409b0bee93ff72b801bcdafaaa4e6c8dc569531d7e7d7606cb4c089c2606d99f5c7bc379cccd4c6979695fe781255ec361708c74cda9667d580c8722eee96
-  data.tar.gz: 59e11ff34787e24df2e4aa6a5cf2590f98adb18594db0f6e18d740cd74a428ac1defb40bb4476ef9366de2abc51ee887c205c14e00a092ca9bffeb09aee01596
+  metadata.gz: 1fc109e80a9808fe201f7aa683fd2c0d9a8c0f35e6a1b2f3f9a0ee5f1f62801f192497c51ce09e69e5a721cb301d564227e543c44f2263aaf1faa3da14215c8c
+  data.tar.gz: 78d53fa4a1bc35af227a4d15553204a31405e89459e2b51a60cad762eba536b0e73f0719e51058ca8df03f328ab6117986f6109a2e01c531bb12c3f0b8453576

data/lib/rbbt/persist.rb

@@ -83,7 +83,7 @@ module Persist
 
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
   def self.load_file(path, type)
-    case (type || "nil").to_sym
+    case (type || :marshal).to_sym
     when :nil
       nil
     when :boolean
@@ -125,7 +125,7 @@ module Persist
 
     return if content.nil?
     
-    case (type || "nil").to_sym
+    case (type || :marshal).to_sym
     when :nil
       nil
     when :boolean
@@ -134,7 +134,8 @@ module Persist
       content.file.seek 0
       Misc.sensiblewrite(path, content.file.read)
     when :tsv
-      Misc.sensiblewrite(path, content.to_s)
+      content = content.to_s if TSV === content
+      Misc.sensiblewrite(path, content)
     when :annotations
       Misc.sensiblewrite(path, Annotated.tsv(content, :all).to_s)
     when :string, :text
@@ -146,10 +147,21 @@ module Persist
       f.close
       content
     when :array
-      if content.empty?
-        Misc.sensiblewrite(path, "")
+      case content
+      when Array
+        if content.empty?
+          Misc.sensiblewrite(path, "")
+        else
+          Misc.sensiblewrite(path, content * "\n" + "\n")
+        end
+      when IO
+        Misc.sensiblewrite(path) do |file|
+          while block = content.read(2048)
+            file.write block
+          end
+        end
       else
-        Misc.sensiblewrite(path, content * "\n" + "\n")
+        Misc.sensiblewrite(path, content.to_s)
       end
     when :marshal_tsv
       Misc.sensiblewrite(path, Marshal.dump(content.dup))
@@ -164,12 +176,67 @@ module Persist
     end
   end
 
+  def self.tee_stream(stream, path, type, callback = nil)
+    file_out, file_in = IO.pipe
+    stream_out, stream_in = IO.pipe
+
+    saver_thread = Thread.new(Thread.current) do |parent|
+      begin
+        Misc.lock(path) do
+          save_file(path, type, file_out)
+        end
+      rescue Exception
+        Log.exception $!
+        parent.raise $!
+      end
+    end
+
+    splitter_thread = Thread.new(Thread.current) do |parent|
+      begin
+        while block = stream.read(2048)
+          begin stream_in.write block; rescue Exception; Log.exception $! end
+        begin file_in.write block; rescue Exception;  Log.exception $! end
+        end
+        file_in.close
+        stream_in.close
+        callback.call if callback
+      rescue Exception
+        Log.exception $!
+        parent.raise $!
+      end
+    end
+
+    class << stream_out
+      attr_accessor :threads
+
+      def join
+        @threads.each{|t| t.join }
+        @threads = []
+      end
+
+      def close
+        join
+        super
+      end
+
+      def read(*args)
+        res = super(*args)
+        join if eof?
+        res
+      end
+    end
+
+    stream_out.threads = [splitter_thread, saver_thread]
+
+    stream_out
+  end
+
   def self.persist(name, type = nil, persist_options = {})
     type ||= :marshal
     persist_options = Misc.add_defaults persist_options, :persist => true
-    other_options = Misc.process_options persist_options, :other
 
     if persist_options[:persist]
+      other_options = Misc.process_options persist_options, :other
       path = persistence_path(name, persist_options, other_options || {})
 
       case 
@@ -276,8 +343,13 @@ module Persist
             Log.medium "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
             res = yield
 
-            if res.nil?
+            case res
+            when nil
               res = load_file(path) unless persist_options[:no_load]
+            when IO
+              res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
+            when TSV::Dumper
+              res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
             else
               Misc.lock(path) do
                 save_file(path, type, res)

data/lib/rbbt/tsv/accessor.rb

@@ -1,5 +1,6 @@
 require 'yaml'
 require 'rbbt/annotations'
+require 'rbbt/tsv/dumper'
 module TSV
 
   TSV_SERIALIZER = YAML
@@ -460,7 +461,7 @@ module TSV
   def options
     options = {}
     ENTRIES.each do |entry|
-      options[entry] = self.send(entry)
+      options[entry.to_sym] = self.send(entry)
     end
     IndiferentHash.setup options
   end
@@ -498,26 +499,31 @@ module TSV
       end
     end
 
-    str = ""
-
-    entry_hash = no_options ? {} : (ENTRIES - ["key_field", "fields"]).collect{|key| [key.to_sym, self.send(key)]}
-    str = TSV.header_lines(key_field, fields, entry_hash)
-
-    with_unnamed do
-      if keys.nil?
-        each do |key, values|
-          key = key.to_s if Symbol === key
-          str << key.to_s
-          str << values_to_s(values)
-        end
-      else
-        keys.zip(values_at(*keys)).each do |key, values|
-          key = key.to_s if Symbol === key
-          str << key.to_s << values_to_s(values)
+    io = TSV::Dumper.stream self do |dumper|
+      dumper.init unless no_options
+      begin
+        if keys
+          keys.each do |key|
+            dumper.add key, self[key]
+          end
+        else
+          with_unnamed do
+            each do |k,v|
+              dumper.add k, v
+            end
+          end
         end
+      rescue
+        Log.exception $!
+        parent.raise $!
       end
+    end
 
+    str = ''
+    while block = io.read(2048)
+      str << block
     end
+
     str
   end
 

data/lib/rbbt/tsv/dumper.rb

@@ -0,0 +1,56 @@
+module TSV
+  class Dumper
+    attr_accessor :in_stream, :stream, :options, :filename
+    def self.stream(options = {}, filename = nil, &block)
+      dumper = TSV::Dumper.new options, filename
+      Thread.new do
+        yield dumper
+        dumper.close
+      end
+      dumper.stream
+    end
+
+    def initialize(options, filename = nil)
+      if TSV  === options
+        @options = options.options.merge(:key_field => options.key_field, :fields => options.fields)
+        @filename ||= options.filename
+      else
+        @options = options
+        @filename = filename
+      end
+      @filename ||= Misc.fingerprint options
+      @stream, @in_stream = IO.pipe
+    end
+
+    def self.values_to_s(values, fields = nil)
+      case values
+      when nil
+        if fields.nil? or fields.empty?
+          "\n"
+        else
+          "\t" << ([""] * fields.length) * "\t" << "\n"
+        end
+      when Array
+        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      else
+        "\t" << values.to_s << "\n"
+      end
+    end
+
+    def init
+      options = @options.dup
+      key_field, fields = Misc.process_options options, :key_field, :fields
+
+      str = TSV.header_lines(key_field, fields, options)
+      @in_stream.puts str
+    end
+
+    def add(k,v)
+      @in_stream << k << TSV::Dumper.values_to_s(v, @options[:fields])
+    end
+
+    def close
+      @in_stream.close
+    end
+  end
+end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -54,7 +54,29 @@ module TSV
       traverse_tsv(obj, options, &block)
     when Hash
       traverse_hash(obj, options, &block)
-    when IO
+    when TSV::Parser
+      callback = Misc.process_options options, :callback
+      if callback
+        obj.traverse(options) do |k,v|
+          res = yield k, v
+          callback.call res
+        end
+      else
+        obj.traverse(options, &block)
+      end
+    when (options[:type] == :array and IO)
+      callback = Misc.process_options options, :callback
+      if callback
+        while not obj.eof?
+          res = yield obj.gets.strip
+          callback.call res
+        end
+      else
+        while not obj.eof?
+          yield obj.gets.strip
+        end
+      end
+    when IO, File
       callback = Misc.process_options options, :callback
       if callback
         TSV::Parser.traverse(obj, options) do |k,v|
@@ -68,8 +90,20 @@ module TSV
       obj.open do |stream|
         traverse_obj(stream, options, &block)
       end
+    when (defined? Step and Step)
+      case obj.result
+      when IO
+        traverse_obj(obj.result, options, &block)
+      when TSV::Dumper
+        traverse_obj(obj.stream, options, &block)
+      else
+        obj.join
+        traverse_obj(obj.path.open, options, &block)
+      end
     when Array
       traverse_array(obj, options, &block)
+    when nil
+      raise "Can not traverse nil object"
     else
       raise "Unknown object for traversal: #{Misc.fingerprint obj }"
     end
@@ -80,7 +114,6 @@ module TSV
 
     q = RbbtThreadQueue.new num
 
-
     if callback
       block = Proc.new do |k,v,mutex|
         v, mutex = nil, v if mutex.nil?
@@ -133,35 +166,60 @@ module TSV
         k,v = value
         obj[k] = v
       end
-    when IO
+    when TSV::Dumper
       return if value.nil?
-      obj << value
+      obj.add *value
+    when IO, StringIO
+      return if value.nil?
+      obj.puts value
     else
       obj << value
     end 
   end
 
+  def self.traverse_run(obj, threads, cpus, options = {}, &block)
+    if threads.nil? and cpus.nil? 
+      traverse_obj obj, options, &block
+    else
+      if threads
+        traverse_threads threads, obj, options, &block 
+      else
+        traverse_cpus cpus, obj, options, &block
+      end
+    end
+  end
+
   def self.traverse(obj, options = {}, &block)
     threads = Misc.process_options options, :threads
     cpus = Misc.process_options options, :cpus
-    into = Misc.process_options options, :into
+    into = options[:into]
+
+    threads = nil if threads and threads.to_i <= 1
+    cpus = nil if cpus and cpus.to_i <= 1
 
     if into
       callback = Proc.new do |e|
         store_into into, e
       end
       options[:callback] = callback
-    end
 
-    if threads.nil? and cpus.nil?
-      traverse_obj obj, options, &block
-    else
-      if threads
-        traverse_threads threads, obj, options, &block 
+      case into
+      when TSV::Dumper, IO, StringIO
+        Thread.new(Thread.current) do |parent|
+          begin
+            traverse_run(obj, threads, cpus, options, &block)
+            into.close 
+          rescue Exception
+            parent.raise $!
+          end
+        end
       else
-        traverse_cpus cpus, obj, options, &block
+        traverse_run(obj, threads, cpus, options, &block)
       end
+
+      into
+    else
+      traverse_run(obj, threads, cpus, options, &block)
     end
-    into
   end
 end

data/lib/rbbt/tsv/parser.rb

@@ -1,7 +1,7 @@
 require 'rbbt/util/cmd'
 module TSV
   class Parser
-    attr_accessor :stream, :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace, :first_line, :stream
+    attr_accessor :stream, :filename, :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight, :take_all, :zipped, :namespace, :first_line, :stream
 
     class SKIP_LINE < Exception; end
     class END_PARSING < Exception; end
@@ -24,7 +24,7 @@ module TSV
       # Process options line
 
       if line and line =~ /^#{@header_hash}: (.*)/
-        options = Misc.string2hash $1
+        options = Misc.string2hash $1.strip
         line = Misc.fixutf8 stream.gets
       end
 
@@ -343,11 +343,16 @@ module TSV
       @sep = Misc.process_options(options, :sep) || "\t"
       @stream = stream
 
+
       header_options = parse_header(stream)
+
       options = header_options.merge options
 
       @type = Misc.process_options(options, :type) || :double
 
+      @filename = Misc.process_options(options, :filename) 
+      @filename ||= stream.filename if stream.respond_to? :filename
+
       @sep2 = Misc.process_options(options, :sep2) || "|"
       @cast = Misc.process_options options, :cast; @cast = @cast.to_sym if String === @cast
       @type ||= Misc.process_options options, :type
@@ -362,6 +367,7 @@ module TSV
       fields = options[:fields]
       fix_fields(options)
 
+      @type = @type.strip.to_sym if String === @type
       case @type
       when :double 
         self.instance_eval do alias get_values get_values_double end
@@ -407,6 +413,8 @@ module TSV
             self.instance_eval do alias add_to_data add_to_data_flat end
           end
         end
+      else
+        raise "Unknown TSV type: #{@type.inspect}"
       end
 
 
@@ -419,10 +427,23 @@ module TSV
       data.key_field = @key_field
       data.fields = @fields
       data.namespace = @namespace
+      data.filename = @filename
       data.cast = @cast if Symbol === @cast
       data
     end
 
+    def annotate(data)
+      setup(data)
+    end
+
+    def options
+      options = {}
+      TSV::ENTRIES.each do |entry|
+        options[entry.to_sym] = self.send(entry) if self.respond_to? entry
+      end
+      IndiferentHash.setup options
+    end
+
     def traverse(options = {})
       monitor, grep, invert_grep, head = Misc.process_options options, :monitor, :grep, :invert_grep, :head
       raise "No block given in TSV::Parser#traverse" unless block_given?
@@ -493,8 +514,8 @@ module TSV
             break
           end
         end
-      ensure
-        stream.close unless stream.closed?
+      #ensure
+      #  stream.close unless stream.closed?
       end
 
       self

data/lib/rbbt/tsv/util.rb

@@ -92,6 +92,7 @@ module TSV
       raise "Cannot get stream from: #{file.inspect}"
     end
   end
+
   def self.get_stream(file, open_options = {})
     case file
     when Path

data/lib/rbbt/util/misc.rb

@@ -31,10 +31,11 @@ end
 Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
 module Misc
 
+
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
     i = 0
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
-    text.split(re).collect do |paragraph|
+      text.split(re).collect do |paragraph|
       i += 1
       str = if i % 2 == 1
               words = paragraph.gsub(/\s+/, "\s").split(" ")
@@ -56,7 +57,7 @@ module Misc
             end
       offset = 0
       str
-    end*""
+      end*""
   end
 
   def self.format_definition_list_item(dt, dd, size = 80, indent = 20, color = :yellow)
@@ -314,6 +315,12 @@ module Misc
       end
     when AnnotatedArray
       "<A: #{fingerprint Annotated.purge(obj)} #{fingerprint obj.info}>"
+    when TSV::Parser
+      "<TSVStream:" + obj.filename + "--" << Misc.fingerprint(obj.options) << ">"
+    when IO
+      "<IO:" + (obj.respond_to?(:filename) ? obj.filename : obj.inspect) + ">"
+    when File
+      "<File:" + obj.path + ">"
     when Array
       if (length = obj.length) > 10
         "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
@@ -347,6 +354,12 @@ module Misc
 
   def self.remove_long_items(obj)
     case
+    when IO === obj
+      remove_long_items("IO: " + obj.filename)
+    when obj.respond_to?(:path)
+      remove_long_items("File: " + obj.path)
+    when TSV::Parser === obj
+      remove_long_items("TSV Stream: " + obj.filename + " -- " << Misc.fingerprint(obj.options))
     when TSV === obj
       remove_long_items((obj.all_fields || []) + obj.keys.sort)
     when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
@@ -1276,6 +1289,8 @@ end
         str << k.to_s << "=>" << v
       when (Array === v and v.length > HASH2MD5_MAX_ARRAY_LENGTH)
         str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "; #{ v.length }]"
+      when TSV::Parser === v
+        str << remove_long_items(v)
       when Array === v
         str << k.to_s << "=>[" << v * "," << "]"
       else

data/lib/rbbt/workflow/accessor.rb

@@ -41,7 +41,10 @@ class Step
 
   def info
     return {} if info_file.nil? or not Open.exists? info_file
-    return @info_cache if @info_cache and File.mtime(info_file) < @info_cache_time
+    begin
+      return @info_cache if @info_cache and File.mtime(info_file) < @info_cache_time
+    rescue Exception
+    end
     begin
       @info_cache = Misc.insist(3, 5, info_file) do
         Misc.insist(2, 2, info_file) do
@@ -66,7 +69,7 @@ class Step
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file) do
       i = info
-      i[key] = value
+      i[key] = value #File === value ? value.filename : value
       @info_cache = i
       Open.write(info_file, INFO_SERIALIAZER.dump(i))
       @info_cache_time = Time.now
@@ -143,7 +146,7 @@ class Step
   def running?
     return nil if not Open.exists? info_file
     return nil if info[:pid].nil?
-    return Misc.pid_exists? info[:pid]
+    return Misc.pid_exists?(p = info[:pid]) && Process.pid != p
   end
 
   def error?
@@ -361,7 +364,10 @@ module Workflow
       if inputs.any? or dependencies.any?
         tagged_jobname = case TAG
                          when :hash
-                           jobname + '_' + Misc.digest((inputs.collect{|i| Misc.fingerprint(i)} * "," + ";" + dependencies.collect{|dep| dep.name } * "\n"))
+                           input_str = ""
+                           input_str << inputs.collect{|i| Misc.fingerprint(i) } * "," 
+                           input_str << ";" << dependencies.collect{|dep| dep.name } * "\n"
+                           jobname + '_' << Misc.digest(input_str)
                          else
                            jobname
                          end

data/lib/rbbt/workflow/step.rb

@@ -9,6 +9,7 @@ class Step
   attr_accessor :path, :task, :inputs, :dependencies, :bindings
   attr_accessor :pid
   attr_accessor :exec
+  attr_accessor :result
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
@@ -54,7 +55,24 @@ class Step
   end
 
   def prepare_result(value, description = nil, info = {})
-    case
+    case 
+    when IO === value
+      begin
+        case @task.result_type
+        when :array
+          array = []
+          while line = value.gets
+            array << line
+          end
+          array
+        when :tsv
+          TSV.open(value)
+        else
+          value.read
+        end
+      ensure
+        value.join if value.respond_to? :join
+      end
     when (not defined? Entity or description.nil? or not Entity.formats.include? description)
       value
     when (Annotated === value and info.empty?)
@@ -70,13 +88,24 @@ class Step
     end
   end
 
-  def exec
+  def _exec
     @exec = true if @exec.nil?
-    result = @task.exec_in((bindings ? bindings : self), *@inputs)
-    prepare_result result, @task.result_description
+    @task.exec_in((bindings ? bindings : self), *@inputs)
+  end
+
+  def exec(no_load=false)
+    @result = _exec
+    @result = @result.stream if TSV::Dumper === @result
+    no_load ? @result : prepare_result(@result, @task.result_description)
   end
 
   def join
+    case @result
+    when IO
+      while @result.read 2048; end
+      @result = nil
+    end
+
     if @pid.nil?
       self
     else
@@ -97,7 +126,7 @@ class Step
 
   def run(no_load = false)
 
-    result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do
+    result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => false do
       if Step === Step.log_relay_step and not self == Step.log_relay_step
         relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
       end
@@ -126,52 +155,65 @@ class Step
 
       set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
-      #Log.info{"#{Log.color :magenta, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]: #{ Log.color :blue, path }"}
       set_info :started, (start_time = Time.now)
       log :started, "#{Log.color :magenta, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
 
-      res = begin
-              exec
-            rescue Aborted
-              log(:error, "Aborted")
-
-              children_pids = info[:children_pids]
-              if children_pids and children_pids.any?
-                Log.medium("Killing children: #{ children_pids * ", " }")
-                children_pids.each do |pid|
-                  Log.medium("Killing child #{ pid }")
-                  begin
-                    Process.kill "INT", pid
-                  rescue Exception
-                    Log.medium("Exception killing child #{ pid }: #{$!.message}")
-                  end
-                end
-              end
-
-              raise $!
+      begin
+        result = _exec
+      rescue Aborted
+        log(:error, "Aborted")
+
+        children_pids = info[:children_pids]
+        if children_pids and children_pids.any?
+          Log.medium("Killing children: #{ children_pids * ", " }")
+          children_pids.each do |pid|
+            Log.medium("Killing child #{ pid }")
+            begin
+              Process.kill "INT", pid
             rescue Exception
-              backtrace = $!.backtrace
+              Log.medium("Exception killing child #{ pid }: #{$!.message}")
+            end
+          end
+        end
 
-              # HACK: This fixes an strange behaviour in 1.9.3 where some
-              # backtrace strings are coded in ASCII-8BIT
-              backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
+        raise $!
+      rescue Exception
+        backtrace = $!.backtrace
 
-              set_info :backtrace, backtrace 
-              log(:error, "#{$!.class}: #{$!.message}")
-              raise $!
-            end
+        # HACK: This fixes an strange behaviour in 1.9.3 where some
+        # backtrace strings are coded in ASCII-8BIT
+        backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
 
-      set_info :done, (done_time = Time.now)
-      set_info :time_elapsed, (time_elapsed = done_time - start_time)
-      log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+        set_info :backtrace, backtrace 
+        log(:error, "#{$!.class}: #{$!.message}")
+        raise $!
+      end
+
+      case result
+      when IO, TSV::Dumper
+        log :streaming, "#{Log.color :magenta, "Streaming task result"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+        class << result
+          attr_accessor :callback
+        end
+        result.callback = Proc.new do
+          set_info :done, (done_time = Time.now)
+          set_info :time_elapsed, (time_elapsed = done_time - start_time)
+          log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+        end
+      else
+        set_info :done, (done_time = Time.now)
+        set_info :time_elapsed, (time_elapsed = done_time - start_time)
+        log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+      end
 
-      res
+      result
     end
 
     if no_load
+      @result = result
       self
     else
-      prepare_result result, @task.result_description, info
+      @result = prepare_result result, @task.result_description, info
     end
   end
 
@@ -179,11 +221,13 @@ class Step
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil?
     @pid = Process.fork do
       begin
-        #trap(:INT) { raise Aborted.new "INT signal recieved" }
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
         FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
         begin
-          run(true)
+          res = run(true)
+          io = res.result  if IO === res.result
+          io = res.result.stream  if TSV::Dumper === res.result
+          while not io.eof?; io.read(2048); end if io
         rescue Aborted
           Log.debug{"Forked process aborted: #{path}"}
           log :aborted, "Aborted"
@@ -212,7 +256,6 @@ class Step
           exit -1
         end
         set_info :pid, nil
-        exit 0
       ensure
         RbbtSemaphore.post_semaphore(semaphore) if semaphore
       end
@@ -224,13 +267,13 @@ class Step
 
   def abort
     @pid ||= info[:pid]
-    if @pid.nil? and info[:forked]
+    if @pid.nil? and info[:forked] 
       Log.medium "Could not abort #{path}: no pid"
       false
     else
       Log.medium "Aborting #{path}: #{ @pid }"
       begin
-        Process.kill("KILL", @pid)
+        Process.kill("KILL", @pid) unless Process.pid == @pid
         Process.waitpid @pid
       rescue Exception
         Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
@@ -254,15 +297,19 @@ class Step
   end
 
   def load
-    raise "Can not load: Step is waiting for proces #{@pid} to finish" if not done?
-    result = Persist.persist "Job", @task.result_type, :file => @path, :check => checks do
-      exec
-    end
-    prepare_result result, @task.result_description, info
+    return prepare_result @result, @task.result_description if @result
+    join if not done?
+    return Persist.load_file(@path, @task.result_type) if @path.exists?
+    exec
   end
 
   def clean
     if Open.exists?(path) or Open.exists?(info_file)
+      begin
+        self.abort if self.running?
+      rescue Exception
+      end
+
       begin
         Open.rm info_file if Open.exists? info_file
         Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'

data/lib/rbbt/workflow.rb

@@ -51,6 +51,12 @@ module Workflow
     end
   end
 
+  def self.installed_workflows
+    self.workflow_dir.glob('**/workflow.rb').collect do |file|
+      File.basename(File.dirname(file))
+    end
+  end
+
   def self.workflow_dir
     @workflow_dir ||= begin
                         case

data/share/rbbt_commands/workflow/list

@@ -10,11 +10,10 @@ workflow_dir = options[:workflow_dir] || Workflow.workflow_dir
 
 Path.setup(workflow_dir) unless Path === workflow_dir
 
-files = workflow_dir.find_all.collect{|p| p.glob("*") }.flatten.select{|f| File.directory? f }
+workflows = Workflow.installed_workflows
 
 if options[:describe]
-  files.each do |file|
-    workflow = File.basename(file)
+  workflows.each do |workflow|
     Workflow.require_workflow workflow
     workflow = Workflow.workflows.select{|w| Misc.camel_case(w.to_s) == Misc.camel_case(workflow)}.first
     puts "# "<<[Misc.camel_case(workflow.to_s), workflow.description] * ": "
@@ -25,8 +24,7 @@ if options[:describe]
 
   end
 else
-  files.each do |file|
-    workflow = File.basename(file)
+  workflows.each do |workflow|
     puts Misc.camel_case(workflow.to_s)
   end
 end

data/share/rbbt_commands/workflow/task

@@ -54,13 +54,32 @@ def SOPT_options(workflow, task)
   sopt_options * ":"
 end
 
+def get_value_stream(value)
+  if value == "-"
+    io = Misc.open_pipe do |sin|
+      while not STDIN.eof? 
+        sin.write STDIN.read(2048)
+      end
+      sin.close
+    end
+  else
+    io = Open.open(value)
+  end
+  class << io
+    attr_accessor :filename
+  end
+  io.filename = value
+  io
+end
+
 def fix_options(workflow, task, job_options)
-  option_types = IndiferentHash.setup workflow.rec_input_types(task.name)
+  input_types = IndiferentHash.setup workflow.rec_input_types(task.name)
+  input_options = IndiferentHash.setup workflow.rec_input_options(task.name)
 
   job_options_cleaned = {}
 
   job_options.each do |name, value|
-    value = case option_types[name].to_sym
+    value = case input_types[name].to_sym
             when :boolean
               TrueClass == value or %w(true TRUE T yes).include? value
             when :float
@@ -68,41 +87,53 @@ def fix_options(workflow, task, job_options)
             when :integer
               value.to_i
             when :text
-              case
-              when value == '-'
-                STDIN.read
-              when (String === value and File.exists?(value) and not File.directory?(value))
-                Open.read(value)
+              if input_options[name] and input_options[name][:stream] and String === value
+                get_value_stream(value)
               else
-                value
-              end
-            when :array
-              if Array === value
-                value
-              else
-                str = case
+                case
                 when value == '-'
                   STDIN.read
-                when (String === value and File.exists?(value))
+                when (String === value and File.exists?(value) and not File.directory?(value))
                   Open.read(value)
                 else
                   value
                 end
-                
-                if $array_separator
-                  str.split(/#{$array_separator}/)
+              end
+            when :array
+              if input_options[name] and input_options[name][:stream] and String === value
+                get_value_stream(value)
+              else
+                if Array === value
+                  value
                 else
-                  str.split(/[,|\s]/)
+                  str = case
+                        when value == '-'
+                          STDIN.read
+                        when (String === value and File.exists?(value))
+                          Open.read(value)
+                        else
+                          value
+                        end
+
+                  if $array_separator
+                    str.split(/#{$array_separator}/)
+                  else
+                    str.split(/[,|\s]/)
+                  end
                 end
               end
             when :tsv
-              case value
-              when TSV
-                value
-              when '-'
-                TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => $array_separator || "|")
+              if input_options[name] and input_options[name][:stream] and String === value
+                TSV::Parser.new(value == '-' ? STDIN : Open.open(value), :filename => value )
               else
-                TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => $array_separator || "|")
+                case value
+                when TSV
+                  value
+                when '-'
+                  TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => $array_separator || "|")
+                else
+                  TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => $array_separator || "|")
+                end
               end
             else
               value
@@ -228,10 +259,8 @@ job = workflow.job(task.name, name, job_options)
 
 # clean job
 if clean 
-  job.abort if job.running?
   job.clean 
   sleep 1
-  job = workflow.job(task.name, name, job_options)
 end
 
 if recursive_clean 
@@ -250,7 +279,7 @@ begin
   end
 
   if do_exec or (job.respond_to?(:is_exec) and job.is_exec)
-    res = job.exec
+    res = job.exec(true)
     case
     when Array === res
       out.puts res * "\n"
@@ -258,6 +287,10 @@ begin
       out.puts res
     when Hash === res
       out.puts res.to_yaml
+    when IO === res
+      while block = res.read(2048)
+        out.write block
+      end
     else
       out.puts res
     end
@@ -281,7 +314,6 @@ begin
     end
 
     job.fork
-
   else
     job.run(true)
     res = job
@@ -289,22 +321,21 @@ begin
 
 
   if options.delete(:provenance)
+    job.join
     pp job.provenance
     exit 0
   end
 
   if options.delete(:printname)
+    job.join if IO === job.result
     puts job.name
     exit 0
-  else
-    Log.low "Job name: #{job.name}" 
   end
 
   if options.delete(:printpath)
+    job.join
     puts job.path
     exit 0
-  else
-    Log.low "Job name: #{job.name}" 
   end
 
   if do_fork
@@ -321,7 +352,6 @@ begin
         time = Time.now - issued
       end
 
-
       space.times do
         Log.clear_line 
       end
@@ -372,7 +402,16 @@ case res
 when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
   out.puts res.load
 when Step
-  out.puts Open.read(res.path) if File.exists? res.path
+  if IO === res.result
+    io = res.result
+    while line = io.gets do
+      out.puts line
+    end
+    io.close
+  else
+    res.join
+    out.puts Open.read(res.path) if File.exists? res.path
+  end
 else
   out.puts res.to_s
 end

data/test/rbbt/test_persist.rb

@@ -37,4 +37,28 @@ class TestPersist < Test::Unit::TestCase
       end
     end
   end
+
+  def test_tsv_dumper
+    TmpFile.with_file do |tmpdir|
+      stream = Persist.persist("Dumper", :tsv, :dir => tmpdir) do
+        dumper = TSV::Dumper.new :key_field => "Field 1", :fields => ["Field 2"], :type => :single
+
+        dumper.init
+        Thread.new do
+          10.times do |i|
+            key = i.to_s
+            dumper.add key, key + " - 2"
+          end
+          dumper.close
+          Thread.exit
+        end
+        dumper
+      end
+
+      while line = stream.gets do
+        puts line
+      end
+
+    end
+  end
 end

data/test/rbbt/util/test_colorize.rb

@@ -4,9 +4,9 @@ require 'test/unit'
 
 class TestColorize < Test::Unit::TestCase
   def test_color_array
-    a = [:red, :red, :blue, :blue, :yellow]
-    a = (0..16).to_a
+    a = (1..16).to_a
 
-    ddd Colorize.distinct(a)
+    assert_equal 16, Colorize.distinct(a).length
+    assert_equal 16, Colorize.distinct(a).compact.uniq.length
   end
 end

data/test/rbbt/workflow/test_step.rb

@@ -159,7 +159,7 @@ class TestStep < Test::Unit::TestCase
         step.run
         while not job.done? do sleep 1 end
         assert_equal "TEST", Open.read(job.file("test"))
-        assert_equal "WRITE", job.messages.last
+        assert job.messages.include? "WRITE"
       end
     end
   end
@@ -176,7 +176,7 @@ class TestStep < Test::Unit::TestCase
         job = step.fork
         while not job.done? do sleep 1 end
         assert_equal "TEST", Open.read(job.file("test"))
-        assert_equal "WRITE", job.messages.last
+        assert job.messages.include? "WRITE"
       end
     end
   end

data/test/test_helper.rb

@@ -20,11 +20,11 @@ class Test::Unit::TestCase
   end
 
   def teardown
-    if defined? Persist
-      FileUtils.rm_rf Path.setup("", 'rbbt').tmp.test.find :user
-      Persist::CONNECTIONS.values.each do |c| c.close end
-      Persist::CONNECTIONS.clear
-    end
+    #if defined? Persist
+    #  FileUtils.rm_rf Path.setup("", 'rbbt').tmp.test.find :user
+    #  Persist::CONNECTIONS.values.each do |c| c.close end
+    #  Persist::CONNECTIONS.clear
+    #end
   end
 
   def datafile_test(file)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.9.12
+  version: 5.10.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-26 00:00:00.000000000 Z
+date: 2014-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -147,6 +147,7 @@ files:
 - lib/rbbt/tsv/attach.rb
 - lib/rbbt/tsv/attach/util.rb
 - lib/rbbt/tsv/change_id.rb
+- lib/rbbt/tsv/dumper.rb
 - lib/rbbt/tsv/excel.rb
 - lib/rbbt/tsv/field_index.rb
 - lib/rbbt/tsv/filter.rb