rbbt-util 5.13.32 → 5.13.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03d6c0cd25cb674bc5fb28731e6c936d32a8468c
-  data.tar.gz: 0a8c5ba0281f806e9d8712e527b011bae576bacf
+  metadata.gz: ac193605daa68792b87762023e090b4f20b51636
+  data.tar.gz: cff3c91f834c396cfbe8c0c9f65cd046aaff8ffc
 SHA512:
-  metadata.gz: 24555145911d44367c1b9ddfe30c96b7e205811b9838508e68faebeb07da384b4fdc1d9bed59a221f97b00e51ba8ea816fccbb7c26517de79433e3306a9e8fbc
-  data.tar.gz: 6b275e80ce80ac94a8067c8f255c50d718fab0717934ff46e5354640951d4f94ae5361cb92d429b2cd14ba3edaea35f5dd26e9748eb4021a53bd1c33f576592b
+  metadata.gz: 53ca452d6922e58d4f447b37c8dbe42580bc72ee470b2400eeb5cfd1e046f29ec42432295670203040fddf95d546e5f83059aee1e3044f501b8b9d9cab3195be
+  data.tar.gz: 882d73bd8b7b3c025ba441af4f42dfc7b047a7f29ca4aa14456e9733e8ce82e863956c987d35a077d6a6a69bd8e0a9189419fc6f7b96d84bc4cae0d7deb34c4d

data/bin/rbbt

@@ -46,18 +46,6 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 --no_lock_id           #{Log.color :yellow, "Do not track lockfiles with ids (for high-througput and high-concurrency"}
 EOF
 
-if options.delete(:no_lock_id)
-  Misc.use_lock_id = false
-end
-
-if mem_dump = options.delete(:dump_mem)
-  require 'rbbt/monitor'
-  Rbbt.dump_memory(mem_dump, Symbol)
-end
-
-if options.delete :nobar
-  ENV["RBBT_NO_PROGRESS"] = "true"
-end
 
 locate = options.delete :locate_file
 
@@ -73,6 +61,19 @@ else
   end
 end
 
+if options.delete(:no_lock_id)
+  Misc.use_lock_id = false
+end
+
+if mem_dump = options.delete(:dump_mem)
+  require 'rbbt/monitor'
+  Rbbt.dump_memory(mem_dump, Symbol)
+end
+
+if options.delete :nobar
+  ENV["RBBT_NO_PROGRESS"] = "true"
+end
+
 if options[:command_dir]
   $rbbt_command_dir = Path.setup(options[:command_dir].dup)
 else

data/lib/rbbt/monitor.rb

@@ -3,7 +3,7 @@ require 'rbbt'
 module Rbbt
 
   LOCK_DIRS = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all +  Rbbt.var.jobs.find_all +
-    Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all 
+    Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all + Rbbt.tmp.sensiblewrite_lock_dir.find_all
 
   SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
 
@@ -31,10 +31,15 @@ module Rbbt
   end
 
   def self.file_time(file)
-    ctime = File.ctime file
-    atime = File.atime file
-    elapsed = Time.now - ctime
-    {:ctime => ctime, :atime => atime, :elapsed => elapsed}
+    info = {}
+    begin
+      info[:ctime] = File.ctime file
+      info[:atime] = File.atime file
+      info[:elapsed] = Time.now - info[:ctime]
+    rescue Exception
+    end
+    info[:ctime] = Time.now - 999
+    info
   end
 
   #{{{ LOCKS
@@ -49,17 +54,18 @@ module Rbbt
   def self.lock_info(dirs = LOCK_DIRS)
     lock_info = {}
     locks(dirs).each do |f|
+      lock_info[f] = {}
       begin
-        i = file_time(f)
+        lock_info[f].merge!(file_time(f))
         if File.size(f) > 0 
           info = Open.open(f) do |s|
             YAML.load(s) 
           end
-          i[:pid] = info[:pid]
-          i[:ppid] = info[:ppid]
+          IndiferentHash.setup(info)
+          lock_info[f][:pid] = info[:pid]
+          lock_info[f][:ppid] = info[:ppid]
         end
-        lock_info[f] = i
-      rescue
+      rescue Exception
         Log.exception $!
       end
     end

data/lib/rbbt/persist.rb

@@ -131,7 +131,7 @@ module Persist
     end
   end
 
-  def self.save_file(path, type, content)
+  def self.save_file(path, type, content, lockfile = nil)
 
     return if content.nil?
 
@@ -139,17 +139,17 @@ module Persist
     when :nil
       nil
     when :boolean
-      Misc.sensiblewrite(path, content ? "true" : "false")
+      Misc.sensiblewrite(path, content ? "true" : "false", :lock => lockfile)
     when :fwt
       content.file.seek 0
-      Misc.sensiblewrite(path, content.file.read)
+      Misc.sensiblewrite(path, content.file.read, :lock => lockfile)
     when :tsv
       content = content.to_s if TSV === content
       Misc.sensiblewrite(path, content)
     when :annotations
-      Misc.sensiblewrite(path, Annotated.tsv(content, :all).to_s)
+      Misc.sensiblewrite(path, Annotated.tsv(content, :all).to_s, :lock => lockfile)
     when :string, :text
-      Misc.sensiblewrite(path, content)
+      Misc.sensiblewrite(path, content, :lock => lockfile)
     when :binary
       content.force_encoding("ASCII-8BIT") if content.respond_to? :force_encoding
       f = File.open(path, 'wb')
@@ -160,38 +160,36 @@ module Persist
       case content
       when Array
         if content.empty?
-          Misc.sensiblewrite(path, "")
+          Misc.sensiblewrite(path, "", :lock => lockfile)
         else
-          Misc.sensiblewrite(path, content * "\n" + "\n")
+          Misc.sensiblewrite(path, content * "\n" + "\n", :lock => lockfile)
         end
       when IO
-        Misc.sensiblewrite(path, content)
+        Misc.sensiblewrite(path, content, :lock => lockfile)
       else
-        Misc.sensiblewrite(path, content.to_s)
+        Misc.sensiblewrite(path, content.to_s, :lock => lockfile)
       end
     when :marshal_tsv
-      Misc.sensiblewrite(path, Marshal.dump(content.dup))
+      Misc.sensiblewrite(path, Marshal.dump(content.dup), :lock => lockfile)
     when :marshal
       dump = Marshal.dump(content)
-      Misc.sensiblewrite(path, [dump].pack("m"))
+      Misc.sensiblewrite(path, [dump].pack("m"), :lock => lockfile)
     when :yaml
-      Misc.sensiblewrite(path, YAML.dump(content))
+      Misc.sensiblewrite(path, YAML.dump(content), :lock => lockfile)
     when :float, :integer, :tsv
-      Misc.sensiblewrite(path, content.to_s)
+      Misc.sensiblewrite(path, content.to_s, :lock => lockfile)
     else
       raise "Unknown persistence: #{ type }"
     end
   end
 
-  def self.tee_stream_thread(stream, path, type, callback = nil, abort_callback = nil)
+  def self.tee_stream_thread(stream, path, type, callback = nil, abort_callback = nil, lockfile = nil)
     file, out = Misc.tee_stream(stream)
 
     saver_thread = Thread.new(Thread.current) do |parent|
       begin
         Thread.current["name"] = "file saver: " + path
-        Misc.lock(path) do
-          save_file(path, type, file)
-        end
+        save_file(path, type, file, lockfile)
       rescue Aborted
         Log.medium "Persist stream thread aborted: #{ Log.color :blue, path }"
         file.abort if file.respond_to? :abort
@@ -209,66 +207,6 @@ module Persist
     out
   end
 
-  def self.tee_stream_pipe(stream, path, type, callback = nil, abort_callback = nil)
-    parent_pid = Process.pid
-    out = Misc.open_pipe true, false do |sin|
-      begin
-        file, out = Misc.tee_stream(stream)
-
-        saver_th = Thread.new(Thread.current, path, file) do |parent,path,file|
-          begin
-            Misc.lock(path) do
-              save_file(path, type, file)
-            end
-            Log.high "Stream pipe saved: #{path}"
-          rescue Aborted
-            Log.medium "Persist stream pipe exception: #{ Log.color :blue, path }"
-            stream.abort if stream.respond_to? :abort
-          rescue Exception
-            Log.medium "Persist stream pipe exception: #{ Log.color :blue, path }"
-            Log.exception $!
-            stream.abort if stream.respond_to? :abort
-            stream.join if stream.respond_to? :join
-            raise $!
-          end
-        end
-
-        tee_th = Thread.new(Thread.current) do |parent|
-          begin
-            while block = out.read(2028)
-              sin.write block
-            end
-          rescue Aborted
-            Log.medium "Tee stream thread aborted"
-            sout.abort if sout.respond_to? :abort
-            sin.abort if sin.respond_to? :abort
-          rescue Exception
-            sin.abort if sin.respond_to? :abort
-            sout.abort if sout.respond_to? :abort
-            Log.exception $!
-            raise $!
-          ensure
-            sin.close unless sin.closed?
-          end
-        end
-        saver_th.join
-        tee_th.join
-      rescue Aborted
-        tee_th.raise Aborted.new if tee_th and tee_th.alive?
-        saver_th.raise Aborted.new if saver_th and saver_th.alive?
-        Kernel.exit! -1
-      rescue Exception
-        tee_th.raise Aborted.new if tee_th and tee_th.alive?
-        saver_th.raise Aborted.new if saver_th and saver_th.alive?
-        Log.exception $!
-        Process.kill :INT, parent_pid
-        Kernel.exit! -1
-      end
-    end
-    stream.close
-    out
-  end
-
   class << self
     alias tee_stream tee_stream_thread 
   end
@@ -280,7 +218,7 @@ module Persist
 
     if stream
       if persist_options[:no_load] == :stream 
-        res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
+        res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil, lockfile)
         res.lockfile = lockfile
 
         raise KeepLocked.new res 
@@ -321,10 +259,11 @@ module Persist
       Open.rm path if Open.exists? path
     end
 
-
     lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
     begin
-      Misc.lock lock_filename do |lockfile|
+      lock_options = Misc.pull_keys persist_options, :lock
+      lock_options = lock_options[:lock] if Hash === lock_options[:lock]
+      Misc.lock lock_filename, lock_options do |lockfile|
 
         Misc.insist do
           if is_persisted?(path, persist_options)
@@ -338,9 +277,7 @@ module Persist
 
         res = get_result(path, type, persist_options, lockfile, &block)
 
-        Misc.lock(path) do
-          save_file(path, type, res)
-        end
+        save_file(path, type, res, lockfile)
 
         return path if persist_options[:no_load]
 
@@ -348,7 +285,11 @@ module Persist
       end
 
     rescue Lockfile::StolenLockError
-      Log.medium "Lockfile stolen: #{path}"
+      begin
+        Log.medium "Lockfile stolen: #{path}"
+        sleep 1 + rand(2)
+      rescue Exception
+      end
       retry
     rescue Exception
       Log.medium "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"

data/lib/rbbt/persist/tsv/adapter.rb

@@ -54,8 +54,8 @@ module Persist
 
     def write_and_read
       lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        mutex.synchronize do
+      mutex.synchronize do
+        Misc.lock(lock_filename, true, :max_age => 600) do
           write if closed? or not write?
           res = begin
                   yield
@@ -69,11 +69,14 @@ module Persist
 
     def write_and_close
       lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        mutex.synchronize do
+      mutex.synchronize do
+        Misc.lock(lock_filename, true) do
           write if closed? or not write?
           res = begin
                   yield
+                rescue Exception
+                  Log.exception $!
+                  raise $!
                 ensure
                   close
                 end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -180,9 +180,14 @@ module TSV
 
     if callback
       TSV::Parser.traverse(io, options) do |k,v|
-        callback.call yield k, v
+        begin
+          callback.call yield k, v
+        ensure
+          bar.tick if bar
+        end
       end
     else
+      options[:monitor] = bar
       TSV::Parser.traverse(io, options, &block)
     end
     join.call if join
@@ -492,6 +497,17 @@ module TSV
     threads = nil if threads and threads.to_i <= 1
     cpus = nil if cpus and cpus.to_i <= 1
 
+    if options[:keys]
+      case options[:keys]
+      when TrueClass
+        options[:type] = :keys
+      when String
+        options[:type] = :keys
+        options[:key_field] = options[:keys]
+        options[:fields] = []
+      end
+    end
+
     bar = Misc.process_options options, :bar
     bar ||= Misc.process_options options, :progress
     max = guess_max(obj)

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -7,6 +7,13 @@ class RbbtProcessQueue
 
       @pid = Process.fork do
         begin
+          Persist::CONNECTIONS.values.each do |db| db.close if db.write? end
+          ObjectSpace.each_object(Mutex) do |m| 
+            begin 
+              m.unlock 
+            rescue ThreadError
+            end if m.locked? 
+          end
           @cleanup.call if @cleanup
           @queue.close_write 
 

data/lib/rbbt/util/log.rb

@@ -133,12 +133,12 @@ module Log
     message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message
 
-    LOG_MUTEX.synchronize do
-      STDERR.puts str
-      Log::LAST.replace "log"
-      logfile.puts str unless logfile.nil?
-      nil
-    end
+    #LOG_MUTEX.synchronize do
+    STDERR.puts str
+    Log::LAST.replace "log"
+    logfile.puts str unless logfile.nil?
+    nil
+    #end
   end
 
   def self.log_obj_inspect(obj, level, file = $stdout)

data/lib/rbbt/util/log/progress.rb

@@ -23,7 +23,7 @@ module Log
       (@ticks * 100) / @max
     end
 
-    def tick
+    def tick(step = nil)
       @ticks += 1
 
       time = Time.now

data/lib/rbbt/util/log/progress/report.rb

@@ -89,12 +89,14 @@ module Log
     def report(io = STDERR)
       if Log::LAST != "progress"
         length = Log::ProgressBar.cleanup_bars
-        print(io, Log.color(:yellow, "--Progress\n"))
         bars = BARS
         bars.sort_by{|b| b.depth }.reverse.each do |bar|
           print(io, Log.color(:yellow ,bar.report_msg) << "\n")
         end
+      else
+        bars = BARS
       end
+      print(io, up_lines(bars.length) << Log.color(:yellow, "--Progress\n") << down_lines(bars.length)) 
       print(io, up_lines(@depth) << report_msg << down_lines(@depth)) 
       @last_time = Time.now
       @last_count = ticks

data/lib/rbbt/util/misc/lock.rb

@@ -8,12 +8,15 @@ module Misc
       Lockfile.suspend = 10
     else
       Log.medium "De-activating lockfile ids"
-      Lockfile.dont_use_lock_id = false
       Lockfile.dont_use_lock_id = true
-      Lockfile.refresh = 2 
-      Lockfile.max_age = 5
-      Lockfile.suspend = 1
+      Lockfile.refresh = 5
+      Lockfile.max_age = 30
+      Lockfile.suspend = 5
     end
+
+    Lockfile.refresh = 3
+    Lockfile.max_age = 10
+    Lockfile.suspend = 2
   end
 
   self.use_lock_id = ENV["RBBT_NO_LOCKFILE_ID"] != "true"
@@ -26,19 +29,19 @@ module Misc
     res = nil
 
     lock_path = File.expand_path(file + '.lock')
-    lockfile = Lockfile.new(lock_path, options)
-
-    lockfile.lock 
+    if options[:lock]
+      lockfile = options[:lock]
+      lockfile.lock unless lockfile.locked?
+    else
+      lockfile = Lockfile.new(lock_path, options)
+      lockfile.lock 
+    end
 
     begin
       res = yield lockfile
-    #rescue Lockfile::StolenLockError
     rescue KeepLocked
       unlock = false
       res = $!.payload
-    rescue Exception
-      lockfile.unlock #if lockfile.locked?
-      raise $!
     ensure
       if unlock 
         begin

data/lib/rbbt/util/misc/pipes.rb

@@ -1,4 +1,13 @@
+require 'rbbt'
+
 module Misc
+  class << self
+    attr_accessor :sensiblewrite_lock_dir
+    
+    def sensible_write_locks
+      @sensiblewrite_locks ||= Rbbt.tmp.sensiblewrite_locks.find
+    end
+  end
 
   class << self
     attr_accessor :sensiblewrite_dir
@@ -112,7 +121,7 @@ module Misc
         stream.abort if stream.respond_to? :abort
         stream_out1.abort if stream_out1.respond_to? :abort
         stream_out2.abort if stream_out2.respond_to? :abort
-        stream.join
+        stream.join if stream.respond_to? :join
         Log.medium "Exception in tee_stream_thread: #{$!.message}"
         raise $!
       end
@@ -199,10 +208,12 @@ module Misc
     str
   end
 
-  def self.sensiblewrite(path, content = nil, &block)
+  def self.sensiblewrite(path, content = nil, options = {}, &block)
+    lock_options = Misc.pull_keys options, :lock
+    lock_options = lock_options[:lock] if Hash === lock_options[:lock]
     return if Open.exists? path
-    tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
-    Misc.lock tmp_path do
+    tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_lock_dir})
+    Misc.lock tmp_path, lock_options do
       return if Open.exists? path
       if not Open.exists? path
         FileUtils.rm_f tmp_path if File.exists? tmp_path
@@ -231,7 +242,7 @@ module Misc
             File.open(tmp_path, 'wb') do |f|  end
           end
 
-          Open.mv tmp_path, path
+          Open.mv tmp_path, path, lock_options
           content.join if content.respond_to? :join
         rescue Aborted
           Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"

data/lib/rbbt/util/open.rb

@@ -153,14 +153,14 @@ module Open
 
   def self.save_content_in_repo(dir, sub_path, content)
     repo = get_repo_from_dir(dir)
-    repo.write_and_read do
+    repo.write_and_close do
       repo[sub_path] = content
     end
   end
 
   def self.remove_from_repo(dir, sub_path, recursive = false)
     repo = get_repo_from_dir(dir)
-    repo.write_and_read do
+    repo.write_and_close do
       if recursive
         repo.outlist repo.range sub_path, true, sub_path.sub(/.$/,('\1'.ord + 1).chr), false
       else
@@ -233,7 +233,7 @@ module Open
     end
   end
 
-  def self.mv(source, target)
+  def self.mv(source, target, options)
     dir_sub_path_source = find_repo_dir(source)
     dir_sub_path_target = find_repo_dir(target)
 
@@ -252,8 +252,8 @@ module Open
     repo_source = get_repo_from_dir(dir_sub_path_source[0])
     repo_target = get_repo_from_dir(dir_sub_path_target[0])
 
-    repo_source.write do
-      repo_target.write do
+    repo_source.write_and_close do
+      repo_target.write_and_close do
         repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
       end
     end

data/lib/rbbt/workflow/step.rb

@@ -35,6 +35,12 @@ class Step
     @mutex = Mutex.new
     @info_mutex = Mutex.new
     @inputs = inputs || []
+    NamedArray.setup @inputs, task.inputs
+  end
+
+  def inputs
+    NamedArray.setup @inputs, task.inputs if task.inputs and not NamedArray === @inputs
+    @inputs
   end
 
   def task_name

data/lib/rbbt/workflow/step/run.rb

@@ -92,7 +92,7 @@ class Step
       @result = self._exec
       @result = @result.stream if TSV::Dumper === @result
     end
-    no_load ? @result : prepare_result(@result, @task.result_description)
+    no_load or ENV["RBBT_NO_STREAM"] ? @result : prepare_result(@result, @task.result_description)
   end
 
   def checks

data/share/rbbt_commands/system/clean

@@ -34,7 +34,7 @@ if locks.any?
   puts
   puts Log.color(:magenta, "Locks:")
   locks.each do |file,info|
-    if force or (info[:pid] and Misc.pid_exists? info[:pid])
+    if force or (info[:pid] and not Misc.pid_exists? info[:pid])
       puts "  Removing #{ file }"
       File.unlink file 
     end
@@ -65,6 +65,9 @@ if sensiblewrites.any?
   end
 end
 
+exit 0 if workflow.nil?
+workflow = nil if workflow == ["all"]
+
 puts
 puts Log.color(:magenta, "# Workflow clean")
 puts
@@ -89,7 +92,8 @@ jobs.each do |file,i|
   status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
 
   status = status.to_s
+  next unless status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status == "" or (force and status == 'noinfo')
   puts "  Removing #{ file } - #{status}" 
-  Step.clean(file) if status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status == ""
+  Step.clean(file) 
 end
 

data/share/rbbt_commands/system/status

@@ -20,6 +20,7 @@ rbbt_usage and exit 0 if options[:help]
 
 workflow, task = ARGV
 
+
 workflow = workflow.split "," if workflow
 task = task.split "," if task
 
@@ -58,11 +59,16 @@ end
 
 puts Log.color(:magenta, "# System report")
 puts
-
+sort_file = Proc.new do |f,i|
+  dir = f.split(/:\_/).first
+  
+  dir + i[:ctime].to_i.to_s
+end
 locks = Rbbt.lock_info
 if locks.any?
   puts Log.color(:magenta, "Locks:")
-  locks.sort_by{|f,i| i[:ctime] }.each do |file,info|
+  #locks.sort_by{|f,i| i[:ctime] }.each do |file,info|
+  locks.sort_by(&sort_file).each do |file,info|
     elapsed, pid, ppid = info.values_at :elapsed, :pid, :ppid
     puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}; ppid: #{ppid}; pid: #{pid_msg pid}")
   end
@@ -90,6 +96,8 @@ if sensiblewrites.any?
   puts
 end
 
+exit 0 if workflow.nil?
+workflow = nil if workflow == ["all"]
 
 puts Log.color(:magenta, "# Workflows")
 
@@ -115,7 +123,6 @@ workflows.sort.each do |workflow,tasks|
                  Marshal.load(f)
                end
              rescue
-               Log.exception $!
                {:status => :noinfo}
              end
 

data/test/rbbt/test_monitor.rb

@@ -7,7 +7,7 @@ class TestMonitor < Test::Unit::TestCase
 end
 
 if __FILE__ == $0
-  jobs = Rbbt.jobs("Structure", "annotate")
+  iii Rbbt.lock_info.select{|k,v| k =~ /.file_repo/}
 
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.32
+  version: 5.13.33
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-05 00:00:00.000000000 Z
+date: 2014-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake