rbbt-util 5.13.31 → 5.13.32

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -3,7 +3,9 @@ require 'tokyocabinet'
 module Persist
 
   module TCAdapter
-    attr_accessor :persistence_path, :tokyocabinet_class, :closed, :writable, :mutex
+    include Persist::TSVAdapter
+
+    attr_accessor :tokyocabinet_class
 
     def self.open(path, write, serializer, tokyocabinet_class = TokyoCabinet::HDB)
       tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
@@ -27,21 +29,6 @@ module Persist
       database
     end
 
-    MAX_CHAR = 255.chr
-
-    def prefix(key)
-      range(key, 1, key + MAX_CHAR, 1)
-    end
-
-    def get_prefix(key)
-      keys = prefix(key)
-      select(:key => keys)
-    end
-
-    def closed?
-      @closed
-    end
-
     def close
       @closed = true
       super
@@ -73,83 +60,129 @@ module Persist
       self
     end
 
-    def write?
-      @writable
-    end
-
-    def read?
-      ! write?
-    end
-
-    def collect
-      res = []
-      each do |key, value|
-        res << if block_given?
-                 yield key, value
-        else
-          [key, value]
-        end
-      end
-      res
-    end
-
-    def delete(key)
-      out(key)
-    end
-
-    def write_and_read
-      lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        @mutex.synchronize do
-          write if @closed or not write?
-          res = begin
-                  yield
-                ensure
-                  read
-                end
-          res
-        end
-      end
-    end
-
-    def write_and_close
-      lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        @mutex.synchronize do
-          write if @closed or not write?
-          res = begin
-                  yield
-                ensure
-                  close
-                end
-          res
-        end
-      end
-    end
-
-    def read_and_close
-      @mutex.synchronize do
-        read if @closed or not read?
-        res = begin
-                yield
-              ensure
-                close
-              end
-        res
-      end
-    end
-
-
-    def merge!(hash)
-      hash.each do |key,values|
-        self[key] = values
-      end
-    end
-
-
-    def range(*args)
-      super(*args) #- TSV::ENTRY_KEYS.to_a
-    end
+    #MAX_CHAR = 255.chr
+
+    #def prefix(key)
+    #  range(key, 1, key + MAX_CHAR, 1)
+    #end
+
+    #def get_prefix(key)
+    #  keys = prefix(key)
+    #  select(:key => keys)
+    #end
+
+    #def closed?
+    #  @closed
+    #end
+
+    #def close
+    #  @closed = true
+    #  super
+    #end
+
+    #def read(force = false)
+    #  return if not write? and not closed and not force
+    #  self.close
+    #  if !self.open(@persistence_path, tokyocabinet_class::OREADER)
+    #    ecode = self.ecode
+    #    raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    #  end
+    #  @writable = false
+    #  @closed = false
+    #  self
+    #end
+
+    #def write(force = true)
+    #  return if write? and not closed and not force
+    #  self.close
+
+    #  if !self.open(@persistence_path, tokyocabinet_class::OWRITER)
+    #    ecode = self.ecode
+    #    raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    #  end
+
+    #  @writable = true
+    #  @closed = false
+    #  self
+    #end
+
+    #def write?
+    #  @writable
+    #end
+
+    #def read?
+    #  ! write?
+    #end
+
+    #def collect
+    #  res = []
+    #  each do |key, value|
+    #    res << if block_given?
+    #             yield key, value
+    #    else
+    #      [key, value]
+    #    end
+    #  end
+    #  res
+    #end
+
+    #def delete(key)
+    #  out(key)
+    #end
+
+    #def write_and_read
+    #  lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
+    #  Misc.lock(lock_filename) do
+    #    @mutex.synchronize do
+    #      write if @closed or not write?
+    #      res = begin
+    #              yield
+    #            ensure
+    #              read
+    #            end
+    #      res
+    #    end
+    #  end
+    #end
+
+    #def write_and_close
+    #  lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
+    #  Misc.lock(lock_filename) do
+    #    @mutex.synchronize do
+    #      write if @closed or not write?
+    #      res = begin
+    #              yield
+    #            ensure
+    #              close
+    #            end
+    #      res
+    #    end
+    #  end
+    #end
+
+    #def read_and_close
+    #  @mutex.synchronize do
+    #    read if @closed or not read?
+    #    res = begin
+    #            yield
+    #          ensure
+    #            close
+    #          end
+    #    res
+    #  end
+    #end
+
+
+    #def merge!(hash)
+    #  hash.each do |key,values|
+    #    self[key] = values
+    #  end
+    #end
+
+
+    #def range(*args)
+    #  super(*args) #- TSV::ENTRY_KEYS.to_a
+    #end
   end
 
 

data/lib/rbbt/persist/tsv.rb

@@ -1,3 +1,5 @@
+require 'rbbt/persist/tsv/adapter'
+
 begin
   require 'rbbt/persist/tsv/tokyocabinet'
 rescue Exception

data/lib/rbbt/persist.rb

@@ -111,16 +111,8 @@ module Persist
         res
       when :marshal
         Open.open(path) do |stream|
-          case stream
-          when StringIO
-            begin
-              Marshal.load(stream) 
-            rescue
-              raise $!
-            end
-          else
-            Marshal.load(stream)
-          end
+          content = stream.read.unpack("m").first
+          Marshal.load(content) 
         end
       when :yaml
         Open.open(path) do |stream|
@@ -180,7 +172,8 @@ module Persist
     when :marshal_tsv
       Misc.sensiblewrite(path, Marshal.dump(content.dup))
     when :marshal
-      Misc.sensiblewrite(path, Marshal.dump(content))
+      dump = Marshal.dump(content)
+      Misc.sensiblewrite(path, [dump].pack("m"))
     when :yaml
       Misc.sensiblewrite(path, YAML.dump(content))
     when :float, :integer, :tsv
@@ -288,36 +281,25 @@ module Persist
     if stream
       if persist_options[:no_load] == :stream 
         res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
+        res.lockfile = lockfile
 
-        ConcurrentStream.setup res do
-          begin
-            lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
-          rescue Exception
-            Log.medium "Lockfile exception: " << $!.message
-          end
-        end
-        res.abort_callback = Proc.new do
-          begin
-            lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
-          rescue Exception
-            Log.medium "Lockfile exception: " << $!.message
-          end
-        end
         raise KeepLocked.new res 
       else
+        stream = res.get_stream if res.respond_to? :get_stream
         begin
-          res = case type
-                when :array
-                  res.read.split "\n"
-                when :tsv
-                  TSV.open(res)
-                else
-                  res.read
-                end
-          res.join if res.respond_to? :join
-          res
+          Open.write(path, stream)
+          Open.open(path) do |stream|
+            case type
+            when :array
+              stream.read.split "\n"
+            when :tsv
+              TSV.open(stream)
+            else
+              stream.read
+            end
+          end
         rescue
-          res.abort if res.respond_to? :abort
+          stream.abort if stream.respond_to? :abort
           raise $!
         end
       end
@@ -326,61 +308,7 @@ module Persist
     end
   end
 
-  def self._get_result(path, type, persist_options, lockfile, &block)
-    res = yield
-
-    if persist_options[:no_load] == :stream 
-      stream = IO === res ? res : res.stream
-      res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
-
-      ConcurrentStream.setup res do
-        begin
-            lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
-        rescue Exception
-          Log.medium "Lockfile exception: " << $!.message
-        end
-      end
-      res.abort_callback = Proc.new do
-        begin
-          lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
-        rescue Exception
-          Log.medium "Lockfile exception: " << $!.message
-        end
-      end
-      raise KeepLocked.new res 
-    end
-
-    case res
-    when IO
-      begin
-        res = case
-              when :array
-                res.read.split "\n"
-              when :tsv
-                TSV.open(res)
-              else
-                res.read
-              end
-        res.join if res.respond_to? :join
-      rescue
-        res.abort if res.respond_to? :abort
-        raise $!
-      end
-    when (defined? TSV and TSV::Dumper)
-      begin
-        io = res.stream
-        res = TSV.open(io)
-        io.join if io.respond_to? :join
-      rescue
-        io.abort if io.respond_to? :abort
-        raise $!
-      end
-    end
-    res
-  end
-
   def self.persist_file(path, type, persist_options, &block)
-
     begin
       if is_persisted?(path, persist_options)
         Log.low "Persist up-to-date: #{ path } - #{Misc.fingerprint persist_options}"
@@ -414,7 +342,9 @@ module Persist
           save_file(path, type, res)
         end
 
-        persist_options[:no_load] ? path : res
+        return path if persist_options[:no_load]
+
+        res
       end
 
     rescue Lockfile::StolenLockError

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -281,11 +281,13 @@ module TSV
     rescue Exception
       Log.medium{"Exception traversing #{stream_name(obj)}"}
       Log.exception $!
-      stream = obj_stream(obj)
-      stream.abort if stream and stream.respond_to? :abort
-      stream = obj_stream(options[:into])
-      stream.abort if stream.respond_to? :abort
-      raise $!
+      begin
+        stream = obj_stream(obj)
+        stream.abort if stream and stream.respond_to? :abort
+        stream = obj_stream(options[:into])
+        stream.abort if stream.respond_to? :abort
+        raise $!
+      end
     end
   end
 
@@ -425,6 +427,8 @@ module TSV
   end
 
   def self.traverse_run(obj, threads, cpus, options = {}, &block)
+    threads = nil if threads == 1
+    cpus = nil if cpus == 1
     if ENV["RBBT_NO_MAP_REDUCE"] == "true" or (threads.nil? and cpus.nil?)
       traverse_obj obj, options, &block
     else

data/lib/rbbt/util/cmd.rb

@@ -202,7 +202,7 @@ module CMD
           Log.log line, stderr if Integer === stderr and log
         end
         serr.close
-        Thread.exit
+        Thread.exit!
       end
 
       #SmartIO.tie sout, pid, cmd, post, in_content, sin, serr

data/lib/rbbt/util/log.rb

@@ -17,15 +17,16 @@ module Log
                    end
 
   def self.ignore_stderr
-    backup_stderr = STDERR.dup
     LOG_MUTEX.synchronize do
-      begin
-        File.open('/dev/null', 'w') do |f|
-          STDERR.reopen(f)
+      backup_stderr = STDERR.dup
+      File.open('/dev/null', 'w') do |f|
+        STDERR.reopen(f)
+        begin
           yield
+        ensure
+          STDERR.reopen backup_stderr
+          backup_stderr.close
         end
-      ensure
-        STDERR.reopen backup_stderr
       end
     end
   end

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -1,5 +1,5 @@
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile
 
   def self.setup(stream, options = {}, &block)
     threads, pids, callback, filename, autojoin = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin
@@ -87,6 +87,7 @@ module ConcurrentStream
 
     @joined = true
     close unless closed?
+    lockfile.unlock if lockfile and lockfile.locked?
   end
 
   def abort_threads

data/lib/rbbt/util/misc/development.rb

@@ -220,4 +220,27 @@ module Misc
       ary.values_at *p
     end
   end
+
+  def self.object_delta(*args)
+    res, delta = nil, nil
+    Thread.exclusive do
+      pre = Set.new
+      delta = Set.new
+
+      GC.start
+      ObjectSpace.each_object(*args) do |o|
+        pre.add o
+      end
+
+      res = yield
+
+      GC.start
+      ObjectSpace.each_object(*args) do |o|
+        delta.add o unless pre.include? o
+      end
+
+    end
+    Log.info "Delta: #{delta.inspect}" 
+    res
+  end
 end

data/lib/rbbt/util/misc/lock.rb

@@ -1,17 +1,22 @@
-if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
-  Lockfile.refresh = false 
-  Lockfile.max_age = 60 * 60 * 5
-  Lockfile.max_age = 15
-  Lockfile.suspend = 10
-else
-  Lockfile.dont_use_lock_id = true
-  Lockfile.refresh = 5 
-  Lockfile.max_age = 60
-  Lockfile.suspend = 15
-end
-
-
 module Misc
+  def self.use_lock_id=(use = true)
+    if use
+      Log.medium "Activating lockfile ids"
+      Lockfile.dont_use_lock_id = false
+      Lockfile.refresh = 20 
+      Lockfile.max_age = 60 * 10
+      Lockfile.suspend = 10
+    else
+      Log.medium "De-activating lockfile ids"
+      Lockfile.dont_use_lock_id = false
+      Lockfile.dont_use_lock_id = true
+      Lockfile.refresh = 2 
+      Lockfile.max_age = 5
+      Lockfile.suspend = 1
+    end
+  end
+
+  self.use_lock_id = ENV["RBBT_NO_LOCKFILE_ID"] != "true"
 
   LOCK_MUTEX = Mutex.new
   def self.lock(file, unlock = true, options = {})

data/lib/rbbt/util/misc/pipes.rb

@@ -83,7 +83,6 @@ module Misc
 
     splitter_thread = Thread.new(Thread.current) do |parent|
       begin
-        filename = stream.respond_to?(:filename)? stream.filename : nil
         skip1 = skip2 = false
         while block = stream.read(2048)
           begin 
@@ -148,7 +147,7 @@ module Misc
     str
   end
 
-  def self.consume_stream(io, in_thread = false)
+  def self.consume_stream(io, in_thread = false, into = nil)
     return if Path === io
     return unless io.respond_to? :read 
     if io.respond_to? :closed? and io.closed?
@@ -163,16 +162,19 @@ module Misc
     else
       Log.medium "Consuming stream #{Misc.fingerprint io}"
       begin
-        while block = io.read(2048)
+        while not io.closed? and block = io.read(2048)
+          into << block if into
         end
         io.join if io.respond_to? :join
+        io.close unless io.closed?
       rescue Aborted
         Log.medium "Consume stream aborted #{Misc.fingerprint io}"
         io.abort if io.respond_to? :abort
+        io.close unless io.closed?
       rescue Exception
         Log.medium "Exception consuming stream: #{Misc.fingerprint io}: #{$!.message}"
         io.abort if io.respond_to? :abort
-        io.join
+        io.close unless io.closed?
         raise $!
       end
     end
@@ -207,20 +209,30 @@ module Misc
         begin
           case
           when block_given?
-            File.open(tmp_path, 'w', &block)
+            File.open(tmp_path, 'wb', &block)
           when String === content
-            File.open(tmp_path, 'w') do |f| f.write content end
+            File.open(tmp_path, 'wb') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
-            File.open(tmp_path, 'w') do |f|  
-              while block = content.read(2048); 
-                f.write block
-              end  
+
+            out = nil
+            PIPE_MUTEX.synchronize do
+              out = File.open(tmp_path, 'wb') 
+              out.sync = true
+              OPEN_PIPE_IN << out
             end
+
+            while block = content.read(2048); 
+              out.write block
+            end  
+
+            content.close
+            out.close
           else
-            File.open(tmp_path, 'w') do |f|  end
+            File.open(tmp_path, 'wb') do |f|  end
           end
 
           Open.mv tmp_path, path
+          content.join if content.respond_to? :join
         rescue Aborted
           Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"
           content.abort if content.respond_to? :abort

data/lib/rbbt/util/open.rb

@@ -146,8 +146,8 @@ module Open
   def self.get_stream_from_repo(dir, sub_path)
     repo = get_repo_from_dir(dir)
     repo.read_and_close do
-      text = repo[sub_path]
-      text.nil? ? nil : StringIO.new(text)
+      content = repo[sub_path]
+      content.nil? ? nil : StringIO.new(content)
     end
   end
 
@@ -367,9 +367,9 @@ module Open
            file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
          else
            io = wget(url, wget_options)
-           add_cache(url, io, wget_options)
-           io.close
-           file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
+           new, save = Misc.tee_stream io
+           add_cache(url, save, wget_options)
+           new
          end
     io = unzip(io)  if ((String === url and zip?(url))  and not options[:noz]) or options[:zip]
     io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]

data/lib/rbbt/util/simpleopt/doc.rb

@@ -106,7 +106,7 @@ module SOPT
 
     if description and not description.empty?
       doc << Log.color(:magenta, "## DESCRIPTION") << "\n\n"
-      doc << Misc.format_paragraph(description)
+      doc << Misc.format_paragraph(description) << "\n\n"
     end
 
     doc << Log.color(:magenta, "## OPTIONS") << "\n\n"

data/lib/rbbt/workflow/step/run.rb

@@ -400,7 +400,8 @@ class Step
     stream = get_stream if @result
     if stream
       begin
-        Misc.consume_stream stream
+        @stream_data = StringIO.new
+        Misc.consume_stream stream, false, @stream_data
       rescue Exception
         self._abort
         raise $!
@@ -409,7 +410,7 @@ class Step
   end
 
   def grace
-    until done? or result or error? or aborted? or streaming?
+    until done? or result or error? or aborted? or streaming? 
       sleep 1 
     end
     self