rbbt-util 5.12.0 → 5.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fea2c562a8e3611c8c9767589ec160a36c1b988d
-  data.tar.gz: 42c868b4354a14c64c9eb5ea45b4d447c3d1c757
+  metadata.gz: b59e7f300db3c25c5e9cd000c369f0ccae3b96e3
+  data.tar.gz: 8a4fd1ba66536df107ae3d23426c79b60801a080
 SHA512:
-  metadata.gz: c85bd1f0ed345e6277daed12a41145134f81914e5d1b678db4676296dcb9a8e1ae1bc66c0b3dcc866dc9cab1ebcb80d50cb7181f4e71d755ecb721e3dffa9f55
-  data.tar.gz: fd38d3d2ef666f8b8b4eae6ce66b8ce45b8c5cb6a57e3bbc28c2900f239495a9f991080026e09dc7fbaca480518095e426f482afcc104d07f656835b80509bf2
+  metadata.gz: 6f4720ab8616fd717fd8d945bed47cfb579b7bf524cd83630b5eb7bdb2fcd4f2058a83b156448001be3dbc488058ce147efd2e14e7bcc866b5c207776b07d39e
+  data.tar.gz: f47fffb35b77b270b0a02e1fda74e67d1cc6fd4b67e7cc12ba0881b679fd5a2478cb2e01ea08bcd4e2766df5becbd21f1054fd60998aa6127ce3ea374c1fec61

data/lib/rbbt/persist.rb

@@ -110,9 +110,13 @@ module Persist
         res.pop
         res
       when :marshal
-        Marshal.load(Open.open(path))
+        Open.open(path) do |stream|
+          Marshal.load(stream)
+        end
       when :yaml
-        YAML.load(Open.open(path))
+        Open.open(path) do |stream|
+          YAML.load(stream)
+        end
       when :float
         Open.read(path).to_f
       when :integer

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -152,6 +152,14 @@ module TSV
       end
     when Array
       traverse_array(obj, options, &block)
+    when String
+      if Open.remote? obj or Misc.is_filename? obj
+        Open.open(obj) do |s|
+          traverse_obj(s, options, &block)
+        end
+      else
+        raise "Can not open obj for traversal #{Misc.fingerprint obj}"
+      end
     when nil
       raise "Can not traverse nil object into #{stream_name(options)}"
     else

data/lib/rbbt/tsv/util.rb

@@ -69,6 +69,8 @@ module TSV
 
   def self.get_filename(file)
     case
+    when (defined? Step and Step === file)
+      file.path
     when String === file
       file
     when file.respond_to?(:filename)
@@ -112,6 +114,12 @@ module TSV
       stream || get_stream(file.join.path)
     when TSV::Dumper
       file.stream
+    when Array
+      Misc.open_pipe do |sin|
+        file.each do |l|
+          sin.puts l
+        end
+      end
     else
       raise "Cannot get stream from: #{file.inspect}"
     end

data/lib/rbbt/util/misc/format.rb

@@ -53,7 +53,7 @@ module Misc
     len = Log.uncolor(dt).length
 
     if indent < 0
-      text = format_paragraph(dd, size, indent.abs+1, 0)
+      text = format_paragraph(dd, size, indent.abs-1, 0)
       text = dt << "\n" << text
     else
       offset = len - indent

data/lib/rbbt/util/misc/inspect.rb

@@ -105,16 +105,6 @@ module Misc
     end
   end
 
-  def self.filename?(string)
-    String === string and string.length > 0 and string.length < 250 and File.exists?(string)
-  end
-
-  def self.is_filename?(string)
-    return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and File.exists? string
-    return false
-  end
-
   def self.digest(text)
     Digest::MD5.hexdigest(text)
   end

data/lib/rbbt/util/misc/pipes.rb

@@ -161,16 +161,30 @@ module Misc
     str
   end
 
-  def self.consume_stream(io)
-    return unless io.respond_to? :read
-    begin
-      while block = io.read(2048)
-        return if io.eof?
-        Thread.pass 
-      end
+  def self.consume_stream(io, in_thread = false)
+    return if Path === io
+    return unless io.respond_to? :read 
+    if io.respond_to? :closed? and io.closed?
       io.join if io.respond_to? :join
-    rescue
-      io.abort if io.respond_to? :abort
+      return
+    end
+    if in_thread
+      Thread.new do
+        consume_stream(io, false)
+      end
+    else
+      begin
+        while block = io.read(2048)
+          return if io.eof?
+          Thread.pass 
+        end
+        io.join if io.respond_to? :join
+      rescue
+        Log.error "Exception consuming stream: #{io.inspect}"
+        ddd caller
+        Log.exception $!
+        io.abort if io.respond_to? :abort
+      end
     end
   end
 
@@ -316,7 +330,6 @@ module Misc
         end
         sizes = parts.collect{|p| p.length }
         last_min = nil
-        count ||= 0
         while lines.compact.any?
           min = keys.compact.sort.first
           str = []
@@ -334,7 +347,6 @@ module Misc
                 parts[i] = p
               end
             else
-              count += 1
               str << [sep * (sizes[i]-1)] if sizes[i] > 0
             end
           end

data/lib/rbbt/util/misc/system.rb

@@ -87,4 +87,14 @@ end
     res
   end
 
+  def self.filename?(string)
+    String === string and string.length > 0 and string.length < 250 and File.exists?(string)
+  end
+
+  def self.is_filename?(string)
+    return true if string.respond_to? :exists
+    return true if String === string and string.length < 265 and File.exists? string
+    return false
+  end
+
 end

data/lib/rbbt/workflow.rb

@@ -3,6 +3,7 @@ require 'rbbt/workflow/task'
 require 'rbbt/workflow/step'
 require 'rbbt/workflow/accessor'
 require 'rbbt/workflow/doc'
+require 'rbbt/workflow/examples'
 
 module Workflow
 

data/lib/rbbt/workflow/accessor.rb

@@ -250,7 +250,8 @@ class Step
     when :tsv
       TSV.open Open.open(file(name)), options
     when :array
-      Open.read(file(name)).split /\n|,\s*/
+      #Open.read(file(name)).split /\n|,\s*/
+      Open.read(file(name)).split "\n"
     when :yaml
       YAML.load(Open.open(file(name)))
     when :marshal
@@ -284,29 +285,6 @@ end
 
 module Workflow
 
-  def self.load_inputs(dir, input_names, input_types)
-    inputs = {}
-    dir = Path.setup(dir.dup)
-    input_names.each do |input|
-      file = dir[input].find
-      Log.debug "Trying #{ input }: #{file}"
-      next unless file.exists?
-
-      case input_types[input]
-      when :tsv, :array, :text
-        Log.debug "Pointing #{ input } to #{file}"
-        inputs[input.to_sym]  = file
-      when :boolean
-        inputs[input.to_sym]  = (file.read.strip == 'true')
-      else
-        Log.debug "Loading #{ input } from #{file}"
-        inputs[input.to_sym]  = file.read.strip
-      end
-
-    end
-    IndiferentHash.setup(inputs)
-  end
-
   def log(status, message = nil, &block)
     Step.log(status, message, nil, &block)
   end

data/lib/rbbt/workflow/examples.rb

@@ -0,0 +1,68 @@
+module Workflow
+  attr_accessor :example_dir
+
+  def example_dir
+    @example_dir ||= self.libdir.examples
+  end
+
+  def examples
+    return {} unless self.libdir.examples.exists?
+    examples = {}
+    example_dir.glob("*/*").each do |example_dir|
+      example = File.basename(example_dir)
+      task_name = File.basename(File.dirname(example_dir))
+      examples[task_name] ||= []
+      examples[task_name] << example
+    end
+    IndiferentHash.setup examples
+    examples
+  end
+
+  def example(task_name, example)
+    tasks[task_name.to_sym].input_types.collect do |input,type|
+      next unless example_dir[task_name][example][input].exists?
+      [input, type, example_dir[task_name][example][input].find]
+    end.compact
+  end
+
+  def self.load_inputs(dir, input_names, input_types)
+    inputs = {}
+    dir = Path.setup(dir.dup)
+    input_names.each do |input|
+      file = dir[input].find
+      Log.debug "Trying #{ input }: #{file}"
+      next unless file.exists?
+
+      case input_types[input]
+      when :tsv, :array, :text
+        Log.debug "Pointing #{ input } to #{file}"
+        inputs[input.to_sym]  = file
+      when :boolean
+        inputs[input.to_sym]  = (file.read.strip == 'true')
+      else
+        Log.debug "Loading #{ input } from #{file}"
+        inputs[input.to_sym]  = file.read.strip
+      end
+
+    end
+    IndiferentHash.setup(inputs)
+  end
+
+  def example_step(task_name, example)
+    inputs = {}
+    example(task_name, example).each do |input,type,file|
+
+      case type
+      when :tsv, :array, :text
+        Log.debug "Pointing #{ input } to #{file}"
+        inputs[input.to_sym]  = file
+      when :boolean
+        inputs[input.to_sym]  = (file.read.strip == 'true')
+      else
+        Log.debug "Loading #{ input } from #{file}"
+        inputs[input.to_sym]  = file.read.strip
+      end
+    end
+    Sequence.job(task_name, example, inputs)
+  end
+end

data/lib/rbbt/workflow/usage.rb

@@ -75,31 +75,26 @@ module Workflow
 
       task.doc(dependencies)
 
-      if self.libdir.examples[task_name].exists?
-        self.libdir.examples[task_name].glob("*").each do |example_dir|
-          example = File.basename(example_dir)
-
-          puts Log.color(:magenta, "Example " << example) + " -- " + Log.color(:blue, example_dir)
-
-          inputs = {}
-
-          task.input_types.each do |input,type|
-            if example_dir[input].exists?
-              case type
-              when :tsv, :array, :text
-                head = example_dir[input].read.split("\n")[0..5].compact * "\n\n"
-                head = head[0..500]
-                puts Misc.format_definition_list_item(input, head).gsub("\n\n","\n")
-              else
-                puts Misc.format_definition_list_item(input, example_dir[input].read)
-              end
+      if self.examples.include? task_name
+          self.examples[task_name].each do |example|
+
+            puts Log.color(:magenta, "Example " << example) + " -- " + Log.color(:blue, example_dir[task_name][example])
+
+            inputs = self.example(task_name, example)
+
+            inputs.each do |input, type, file|
+                case type
+                when :tsv, :array, :text
+                  head = file.read.split("\n")[0..5].compact * "\n\n"
+                  head = head[0..500]
+                  puts Misc.format_definition_list_item(input, head, 100, -1).gsub(/\n\s*\n/,"\n")
+                else
+                  puts Misc.format_definition_list_item(input, file.read)
+                end
             end
+            puts
           end
-
-          puts
         end
-
       end
-    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.12.0
+  version: 5.12.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -207,6 +207,7 @@ files:
 - lib/rbbt/workflow/annotate.rb
 - lib/rbbt/workflow/definition.rb
 - lib/rbbt/workflow/doc.rb
+- lib/rbbt/workflow/examples.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/run.rb