rbbt-util 5.14.0 → 5.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c1f84fdf126a66b0e8cf03cd96237dd4625c5d7
-  data.tar.gz: 6e9b9ed9bc339fedaa99f1a96d8f537692ebfbb6
+  metadata.gz: fa73fffbfca850f0304a250527157c35aef1dcb5
+  data.tar.gz: aab332c2a7b11e9d7a207f1640af194dc54a9eb1
 SHA512:
-  metadata.gz: 7879d8241fc10774522d28484ae1cf37bd935067038b1f9e370a2ff4af51adb9f7f183b43c7852694ce4e8b44282ab937b3744b7d8d35072a86e3ccbfb51c48e
-  data.tar.gz: 12ecd4f1f116f4125bb9f4159ab4556910b6e31d0fc3a351c94667758507296914445091575fc8946c975cc305da957949fa79c25913c990e477aed56506e0a0
+  metadata.gz: 231c27fb1f1b1f81569c0613c4e3b5b61d5d0873bb085671421962934802be074c4a94155bd251d59a37d3098ce2a3729febc759c0320eefa453f9c731faf907
+  data.tar.gz: 394a55fe8949acea897c028106ac47bff2f3f3828a9fab1b0fc44446c19ce0c6d40ba83d5da4c9bba8709e35d425741984ff04d643c3dd270cb9a4fb89a5d9e6

data/bin/rbbt

@@ -71,7 +71,7 @@ if mem_dump = options.delete(:dump_mem)
   Rbbt.dump_memory(mem_dump, Symbol)
 end
 
-if options.delete :stream
+if options.delete :nostream
   ENV["RBBT_NO_STREAM"] = "true"
 end
 

data/lib/rbbt/annotations/annotated_array.rb

@@ -37,22 +37,33 @@ module AnnotatedArray
     pos = 0
     super do |value|
 
-      if value.nil?
+      case value
+      when Array
+        value = value.dup if value.frozen?
+
+        value = annotate(value)
+
+        value.extend AnnotatedArray if Array === value and Annotated === value
+
+        value.container       = self
+        value.container_index = pos
+
+        pos += 1
 
         block.call value
-      else
+      when String
 
         value = value.dup if value.frozen?
 
         value = annotate(value)
 
-        value.extend AnnotatedArray if Array === value and Annotated === value
-
         value.container       = self
         value.container_index = pos
 
         pos += 1
 
+        block.call value
+      else
         block.call value
       end
     end

data/lib/rbbt/persist.rb

@@ -132,7 +132,6 @@ module Persist
   end
 
   def self.save_file(path, type, content, lockfile = nil)
-
     return if content.nil?
 
     case (type || :marshal).to_sym
@@ -145,7 +144,7 @@ module Persist
       Misc.sensiblewrite(path, content.file.read, :lock => lockfile)
     when :tsv
       content = content.to_s if TSV === content
-      Misc.sensiblewrite(path, content)
+      Misc.sensiblewrite(path, content, :lock => lockfile)
     when :annotations
       Misc.sensiblewrite(path, Annotated.tsv(content, :all).to_s, :lock => lockfile)
     when :string, :text
@@ -219,8 +218,8 @@ module Persist
 
     if stream
       if persist_options[:no_load] == :stream 
-        res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
-        res.lockfile = lockfile
+        res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil, lockfile)
+        #res.lockfile = lockfile
 
         raise KeepLocked.new res 
       else
@@ -265,7 +264,6 @@ module Persist
       lock_options = Misc.pull_keys persist_options, :lock
       lock_options = lock_options[:lock] if Hash === lock_options[:lock]
       Misc.lock lock_filename, lock_options do |lockfile|
-
         Misc.insist do
           if is_persisted?(path, persist_options)
             Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"

data/lib/rbbt/tsv/dumper.rb

@@ -1,6 +1,6 @@
 module TSV
   class Dumper
-    attr_accessor :in_stream, :stream, :options, :filename
+    attr_accessor :in_stream, :stream, :options, :filename, :sep
     def self.stream(options = {}, filename = nil, &block)
       dumper = TSV::Dumper.new options, filename
       Thread.new(Thread.current) do |parent|
@@ -22,18 +22,19 @@ module TSV
       @stream, @in_stream = Misc.pipe
     end
 
-    def self.values_to_s(values, fields = nil)
+    def self.values_to_s(values, fields = nil, sep = "\t")
+      sep = "\t" if sep.nil?
       case values
       when nil
         if fields.nil? or fields.empty?
           "\n"
         else
-          "\t" << ([""] * fields.length) * "\t" << "\n"
+          sep + ([""] * fields.length) * sep << "\n"
         end
       when Array
-        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+        sep + values.collect{|v| Array === v ? v * "|" : v} * sep << "\n"
       else
-        "\t" << values.to_s << "\n"
+        sep + values.to_s << "\n"
       end
     end
 
@@ -49,9 +50,10 @@ module TSV
 
     def add(k,v)
       @fields ||= @options[:fields]
+      @sep ||= @options[:sep]
       begin
         Thread.pass while IO.select(nil, [@in_stream],nil,1).nil?
-        @in_stream << k << TSV::Dumper.values_to_s(v, @fields)
+        @in_stream << k << TSV::Dumper.values_to_s(v, @fields, @sep)
       rescue IOError
       rescue Exception
         raise $!

data/lib/rbbt/tsv/parser.rb

@@ -544,6 +544,7 @@ module TSV
             stream.abort if stream.respond_to? :abort
             raise $!
           rescue Exception
+            Log.exception $!
             Log.error "Exception parsing #{Misc.fingerprint stream}: #{$!.message}"
             stream.abort $! if stream.respond_to? :abort
             raise $!

data/lib/rbbt/tsv/stream.rb

@@ -175,4 +175,15 @@ module TSV
 
     out
   end
+
+  def self.stream_flat2double(stream, options = {})
+    parser = TSV::Parser.new stream
+    dumper_options = parser.options.merge(options).merge(:type => :double)
+    dumper = TSV::Dumper.new dumper_options
+    dumper.init
+    TSV.traverse parser, :into => dumper do |key,values|
+      [key, [values]]
+    end
+    dumper.stream
+  end
 end

data/lib/rbbt/tsv/util.rb

@@ -82,6 +82,8 @@ module TSV
 
   def self.get_stream(file, open_options = {})
     case file
+    when TSV::Parser
+      file.stream
     when Path
       file.open(open_options)
     when IO, StringIO

data/lib/rbbt/util/log/progress/report.rb

@@ -82,7 +82,11 @@ module Log
       str = Log.color :magenta, desc
       return str << " " << Log.color(:yellow, "waiting") if @ticks == 0
       str << " " << thr_msg
-      str << Log.color(:blue, " -- ") << eta_msg  if max
+      if max
+        str << Log.color(:blue, " -- ") << eta_msg 
+      else
+        str << Log.color(:blue, " -- ") << ticks.to_s
+      end
       str
     end
 

data/lib/rbbt/util/misc/lock.rb

@@ -22,15 +22,20 @@ module Misc
   self.use_lock_id = ENV["RBBT_NO_LOCKFILE_ID"] != "true"
 
   LOCK_MUTEX = Mutex.new
+  #def self.lock(file, unlock = true, options = {})
   def self.lock(file, unlock = true, options = {})
+    unlock, options = true, unlock if Hash === unlock
     return yield if file.nil?
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
 
     res = nil
 
-    if options[:lock]
+    case options[:lock]
+    when Lockfile
       lockfile = options[:lock]
       lockfile.lock unless lockfile.locked?
+    when FalseClass
+      unlock = false
     else
       lock_path = File.expand_path(file + '.lock')
       lockfile = Lockfile.new(lock_path, options)

data/lib/rbbt/util/misc/pipes.rb

@@ -214,9 +214,9 @@ module Misc
 
   def self.sensiblewrite(path, content = nil, options = {}, &block)
     force = Misc.process_options options, :force
-    lock_options = Misc.pull_keys options, :lock
-    lock_options = lock_options[:lock] if Hash === lock_options[:lock]
     return if Open.exists? path and not force
+    lock_options = Misc.pull_keys options.dup, :lock
+    lock_options = lock_options[:lock] if Hash === lock_options[:lock]
     tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
     tmp_path_lock = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_lock_dir})
     Misc.lock tmp_path_lock, lock_options do
@@ -241,7 +241,11 @@ module Misc
           File.open(tmp_path, 'wb') do |f|  end
         end
 
-        Open.mv tmp_path, path, lock_options
+        begin
+          Open.mv tmp_path, path, lock_options
+        rescue
+          raise $! unless File.exists? path
+        end
         content.join if content.respond_to? :join
       rescue Aborted
         Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"

data/lib/rbbt/util/open.rb

@@ -279,7 +279,7 @@ module Open
       repo = get_repo_from_dir(dir)
       Misc.lock_in_repo(repo, sub_path, &block)
     else
-      Misc.lock(file, true, options, &block)
+      Misc.lock(file, options, &block)
     end
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -11,6 +11,7 @@ class Step
   end
   
   def self.wait_for_jobs(jobs)
+    jobs = [jobs] if Step === jobs
     begin
       threads = []
       jobs.each do |j| threads << Thread.new{j.join} end
@@ -112,7 +113,7 @@ class Step
       i = info(false)
       i[key] = value 
       @info_cache = i
-      Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true)
+      Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
       @info_cache_time = Time.now
       value
     end

data/lib/rbbt/workflow/step/run.rb

@@ -15,9 +15,9 @@ class Step
           Log.medium "Not duplicating stream #{ Misc.fingerprint(stream) }"
           STREAM_CACHE[stream] = stream
         when File
-          if Open.exists? current.filename 
+          if Open.exists? current.path 
             Log.medium "Reopening file #{ Misc.fingerprint(current) }"
-            Open.open(current.filename)
+            Open.open(current.path)
           else
             Log.medium "Duplicating file #{ Misc.fingerprint(current) } #{current.inspect}"
             Misc.dup_stream(current)
@@ -131,7 +131,7 @@ class Step
     @seen.each do |dependency|
       next if dependency == self
       next unless dependencies.include? dependency
-      dependency.relay_log self
+      #dependency.relay_log self
       dependency.dup_inputs
     end
 
@@ -235,9 +235,11 @@ class Step
             ConcurrentStream.setup stream do
               begin
                 if status != :done
-                  set_info :done, (done_time = Time.now)
-                  set_info :time_elapsed, (time_elapsed = done_time - start_time)
-                  log :done, "#{Log.color :magenta, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
+                  Misc.insist do
+                    set_info :done, (done_time = Time.now)
+                    set_info :time_elapsed, (time_elapsed = done_time - start_time)
+                    log :done, "#{Log.color :magenta, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
+                  end
                 end
               rescue
                 Log.exception $!

data/test/rbbt/tsv/test_stream.rb

@@ -175,4 +175,25 @@ row1 A B C
     assert_equal ["A", "B", "C", ""], tsv["row1"]
     assert_equal ["AA", "BB", "CC", ""], tsv["row2"]
   end
+
+  def test_flat2double
+    text1=<<-EOF
+#: :sep= #:type=:flat
+#Row LabelA
+row1 A AA AAA
+row2 a aa aaa
+    EOF
+
+    text2=<<-EOF
+#: :sep= #:type=:double
+#Row LabelA
+row1 A|AA|AAA
+row2 a|aa|aaa
+    EOF
+    s1 = StringIO.new text1
+    s2 = TSV.stream_flat2double(s1, :sep => " ")
+
+    assert_equal text2, s2.read
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.0
+  version: 5.14.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-19 00:00:00.000000000 Z
+date: 2014-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake