rbbt-util 5.11.9 → 5.12.0

data/lib/rbbt/util/misc/system.rb

@@ -0,0 +1,90 @@
+require 'net/smtp'
+
+module Misc
+
+  def self.hostname
+    @hostanem ||= `hostname`.strip
+  end
+
+  def self.pid_exists?(pid)
+    return false if pid.nil?
+    begin
+      Process.getpgid(pid.to_i)
+      true
+    rescue Errno::ESRCH
+      false
+    end
+  end
+
+  def self.send_email(from, to, subject, message, options = {})
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, :from_alias => nil, :to_alias => nil, :server => 'localhost', :port => 25, :user => nil, :pass => nil, :auth => :login
+
+    server, port, user, pass, from_alias, to_alias, auth = Misc.process_options options, :server, :port, :user, :pass, :from_alias, :to_alias, :auth
+
+    msg = <<-END_OF_MESSAGE
+From: #{from_alias} <#{from}>
+To: #{to_alias} <#{to}>
+Subject: #{subject}
+
+#{message}
+END_OF_MESSAGE
+
+Net::SMTP.start(server, port, server, user, pass, auth) do |smtp|
+  smtp.send_message msg, from, to
+end
+  end
+
+  def self.env_add(var, value, sep = ":", prepend = true)
+    ENV[var] ||= ""
+    return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
+      if prepend
+        ENV[var] = value + sep + ENV[var]
+      else
+        ENV[var] += sep + ENV[var]
+      end
+  end
+
+  def self.path_relative_to(basedir, path)
+    path = File.expand_path(path) unless path[0] == "/"
+    basedir = File.expand_path(basedir) unless basedir[0] == "/"
+
+    if path.index(basedir) == 0
+      if basedir[-1] == "/"
+        return path[basedir.length..-1]
+      else
+        return path[basedir.length+1..-1]
+      end
+    else
+      return nil
+    end
+  end
+
+  def self.common_path(dir, file)
+    file = File.expand_path file
+    dir = File.expand_path dir
+
+    return true if file == dir
+    while File.dirname(file) != file
+      file = File.dirname(file)
+      return true if file == dir
+    end
+
+    return false
+  end
+
+  # WARN: probably not thread safe...
+  def self.in_dir(dir)
+    old_pwd = FileUtils.pwd
+    res = nil
+    begin
+      FileUtils.mkdir_p dir unless File.exists? dir
+      FileUtils.cd dir
+      res = yield
+    ensure
+      FileUtils.cd old_pwd
+    end
+    res
+  end
+
+end

data/lib/rbbt/util/tar.rb

@@ -43,13 +43,6 @@ module Misc
 
     string = tar.string
 
-    #chunk_size = 1024
-    #Zlib::GzipWriter.wrap(gz) do |writer|
-    #  while chunk = tar.read(chunk_size) do
-    #    writer.write chunk
-    #  end
-    #  writer.close
-    #end
     z = Zlib::GzipWriter.new(gz)
     z.write string
     z.close

data/lib/rbbt/workflow/accessor.rb

@@ -49,9 +49,9 @@ class Step
         end
 
         begin
-          @info_cache = Misc.insist(2, 1, info_file) do
-            Misc.insist(2, 0.5, info_file) do
-              Misc.insist(3, 0.1, info_file) do
+          @info_cache = Misc.insist(2, 3, info_file) do
+            Misc.insist(2, 1, info_file) do
+              Misc.insist(3, 0.2, info_file) do
                 Open.open(info_file) do |file|
                   INFO_SERIALIAZER.load(file) || {}
                 end

data/lib/rbbt/workflow/step/run.rb

@@ -2,6 +2,47 @@ class Step
 
   attr_reader :stream
 
+  STREAM_CACHE = {}
+  STREAM_CACHE_MUTEX = Mutex.new
+  def self.dup_stream(stream)
+    case stream
+    when IO, File
+      return stream if stream.closed?
+      STREAM_CACHE_MUTEX.synchronize do
+        if STREAM_CACHE[stream].nil?
+          STREAM_CACHE[stream] = stream
+        else
+          Misc.dup_stream(STREAM_CACHE[stream])
+        end
+      end
+    when TSV::Dumper, TSV::Parser
+      stream = stream.stream
+      return stream if stream.closed?
+
+      STREAM_CACHE_MUTEX.synchronize do
+        if STREAM_CACHE[stream].nil?
+          STREAM_CACHE[stream] = stream
+        else
+          Misc.dup_stream(STREAM_CACHE[stream])
+        end
+      end
+    else
+      stream
+    end
+  end
+
+  def self.purge_stream_cache
+    return
+    STREAM_CACHE_MUTEX.synchronize do
+      STREAM_CACHE.collect{|k,s| 
+        Thread.new do
+          Misc.consume_stream s
+        end
+      }
+      STREAM_CACHE.clear
+    end
+  end
+
   def get_stream
     @mutex.synchronize do
       @stream = begin
@@ -12,9 +53,16 @@ class Step
     end
   end
 
+  def dup_inputs
+    @inputs.collect do |input|
+      Step.dup_stream input
+    end
+  end
+
   def _exec
     @exec = true if @exec.nil?
-    @task.exec_in((bindings ? bindings : self), *@inputs)
+    #@task.exec_in((bindings ? bindings : self), *@inputs)
+    @task.exec_in((bindings ? bindings : self), *dup_inputs)
   end
 
   def exec(no_load=false)
@@ -299,26 +347,32 @@ class Step
 
     join_stream
 
-    return if not Open.exists? info_file
+    return self if not Open.exists? info_file
+
+    return self if info[:joined]
     pid = @pid 
 
     Misc.insist [0.1, 0.2, 0.5, 1] do
       pid ||= info[:pid]
     end
 
-    if pid.nil?
-      dependencies.each{|dep| dep.join }
-      self
-    else
-      begin
-        Log.debug{"Waiting for pid: #{pid}"}
-        Process.waitpid pid 
-      rescue Errno::ECHILD
-        Log.debug{"Process #{ pid } already finished: #{ path }"}
-      end if Misc.pid_exists? pid
-      pid = nil
-      dependencies.each{|dep| dep.join }
-      self
+    begin
+      if pid.nil?
+        dependencies.each{|dep| dep.join }
+        self
+      else
+        begin
+          Log.debug{"Waiting for pid: #{pid}"}
+          Process.waitpid pid 
+        rescue Errno::ECHILD
+          Log.debug{"Process #{ pid } already finished: #{ path }"}
+        end if Misc.pid_exists? pid
+        pid = nil
+        dependencies.each{|dep| dep.join }
+        self
+      end
+    ensure
+      set_info :joined, true
     end
     self
   end

data/lib/rbbt/workflow/task.rb

@@ -29,15 +29,13 @@ module Task
     end
   end
 
-  def param_options
-  end
-
   def take_input_values(input_values)
     return [] if @inputs.nil?
     values = []
+    defaults = IndiferentHash.setup(@input_defaults || {})
     @inputs.each do |input|
       value = input_values[input]
-      value = IndiferentHash.setup(@input_defaults || {})[input] if value.nil?
+      value = defaults[input] if value.nil?
       values << value
     end
     values
@@ -46,7 +44,11 @@ module Task
   def exec(*args)
     case
     when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
-      self.call *take_input_values(IndiferentHash.setup(args.first))
+      begin
+        self.call *take_input_values(IndiferentHash.setup(args.first))
+      ensure
+        purge_stream_cache
+      end
     else
       self.call *args
     end

data/test/rbbt/tsv/test_stream.rb

@@ -0,0 +1,92 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/stream'
+require 'rbbt'
+
+class TestStream < Test::Unit::TestCase
+  def test_collapse_stream
+    text=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row1 A B C
+row1 a b c
+row2 AA BB CC
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    tsv = TSV.open TSV.collapse_stream(s)
+    assert_equal ["A", "a"], tsv["row1"][0]
+    assert_equal ["BB", "bb"], tsv["row2"][1]
+  end
+
+  def test_paste_stream
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row1 A B C
+row2 AA BB CC
+row3 AAA BBB CCC
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb 
+row1 a b
+row2 aa bb
+row3 aaa bbb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row LabelC
+row1 c
+row2 cc
+row3 ccc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list)
+    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_sort
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+row3 AAA BBB CCC
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb 
+row1 a b
+row3 aaa bbb
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row1 c
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
+  end
+end

data/test/rbbt/tsv/test_util.rb

@@ -18,12 +18,10 @@ row2    A    B    Id3
 
       assert_equal 2, TSV.field_match_counts(tsv, ["a","A","a","b","Id3"])["ValueA"]
       assert_equal nil, TSV.field_match_counts(tsv, ["ValueA"])["ValueA"]
-
     end
-  
   end
 
-  def test_marshal
+  def _test_marshal
    content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2

data/test/rbbt/util/misc/test_pipes.rb

@@ -0,0 +1,79 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../test_helper')
+require 'test/unit'
+require 'rbbt/tsv'
+require 'rbbt/util/misc'
+
+class TestMiscPipes < Test::Unit::TestCase
+
+  def test_collapse_stream
+    text=<<-EOF
+row1 A B C
+row1 a b c
+row2 AA BB CC
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    tsv = TSV.open Misc.collapse_stream(s,nil, " "), :sep => " " 
+    assert_equal ["A", "a"], tsv["row1"][0]
+    assert_equal ["BB", "bb"], tsv["row2"][1]
+  end
+
+  def test_paste_stream
+    text1=<<-EOF
+row1 A B C
+row2 AA BB CC
+row3 AAA BBB CCC
+    EOF
+
+    text2=<<-EOF
+row1 a b
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+row1 c
+row2 cc
+row3 ccc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open Misc.paste_streams([s1,s2,s3],nil, " "), :sep => " ", :type => :list
+    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
+  end
+
+  def test_sort_stream
+    text =<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+    s = StringIO.new text
+    sorted = Misc.sort_stream(s)
+    assert_equal %w(#: #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(#: #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
+  def test_dup_stream
+    text =<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+
+    TmpFile.with_file(text) do |tmp|
+      io = Open.open(tmp)
+      dup = Misc.dup_stream(io)
+      assert_equal text, dup.read
+      assert_equal text, io.read
+    end
+  end
+end