rbbt-util 5.13.30 → 5.13.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 547e2309e238341b65561d216fc6324b85a13165
-  data.tar.gz: b7865ac59ad7bed3894c9fbb24ae3bd8da15f38d
+  metadata.gz: 22e6c0ab1705d1790142288b6bc9943665830554
+  data.tar.gz: 7873db2daa7dafc6ee9b8e1ba9ac12191c8c5d8b
 SHA512:
-  metadata.gz: 98235141db97b051c8c5f6f4c3125937f2ba427407f960de9abc212662578e553024062289feb7f90bd65e763f6084fc4eb33b39b4d52bc743bdca8dd5a0e3ef
-  data.tar.gz: 5cfd97fbd116dbd802c99d477da00ce7b852531e41e7c5f59ea8c84cbf0d07e2bb67a0dd57c5efc1a0a1b938b8400f664567b767a9215a7f7f3093944591ef7f
+  metadata.gz: 07dee96dd2fabee74fbf1ae2852a657f16f30f16c818c092ebd352a6769b080155881a255a48098ebcf749efa95abb13cd9e9da82cecdbdec1a0e415be0c1940
+  data.tar.gz: 547fd0998e00974bc5ecd8bc3ba1b8a98db556cfb48d8c2e2fb0e327eeaaa9af6023112d3ebfdde613911cc459d54c23c60221dc5f55ce6ac40e288d7b33a027

data/lib/rbbt/monitor.rb

@@ -2,7 +2,9 @@ require 'rbbt'
 
 module Rbbt
 
-  LOCK_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all + Rbbt.tmp.tsv_open_locks.find_all + Rbbt.var.jobs.find_all 
+  LOCK_DIRS = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all +  Rbbt.var.jobs.find_all +
+    Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all 
+
   def self.locks(dirs = LOCK_DIRS)
     dirs.collect do |dir|
       dir.glob("**/*.lock")

data/lib/rbbt/persist.rb

@@ -18,7 +18,7 @@ module Persist
     attr_accessor :lock_dir
     
     def lock_dir
-      @lock_dir ||= Rbbt.tmp.tsv_open_locks.find
+      @lock_dir ||= Rbbt.tmp.persist_locks.find
     end
   end
 
@@ -282,11 +282,13 @@ module Persist
 
   def self.get_result(path, type, persist_options, lockfile, &block)
     res = yield
+    stream = res if IO === res
+    stream = res.stream if res.respond_to? :stream
+
+    if stream
+      if persist_options[:no_load] == :stream 
+        res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
 
-    if persist_options[:no_load] == :stream 
-      case res
-      when IO
-        res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil, res.respond_to?(:abort_callback)? res.abort_callback : nil)
         ConcurrentStream.setup res do
           begin
             lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
@@ -302,27 +304,50 @@ module Persist
           end
         end
         raise KeepLocked.new res 
-      when TSV::Dumper
-        stream = res.stream
-        res = tee_stream(stream, path, type)
-        ConcurrentStream.setup res do
-          begin
-            stream.callback
-            lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
-          rescue Exception
-            Log.medium "Lockfile exception: " << $!.message
-          end
+      else
+        begin
+          res = case type
+                when :array
+                  res.read.split "\n"
+                when :tsv
+                  TSV.open(res)
+                else
+                  res.read
+                end
+          res.join if res.respond_to? :join
+          res
+        rescue
+          res.abort if res.respond_to? :abort
+          raise $!
         end
-        res.abort_callback = Proc.new do
-          begin
-            stream.abort
+      end
+    else
+      res
+    end
+  end
+
+  def self._get_result(path, type, persist_options, lockfile, &block)
+    res = yield
+
+    if persist_options[:no_load] == :stream 
+      stream = IO === res ? res : res.stream
+      res = tee_stream(stream, path, type, stream.respond_to?(:callback)? stream.callback : nil, stream.respond_to?(:abort_callback)? stream.abort_callback : nil)
+
+      ConcurrentStream.setup res do
+        begin
             lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
-          rescue Exception
-            Log.medium "Lockfile exception: " << $!.message
-          end
+        rescue Exception
+          Log.medium "Lockfile exception: " << $!.message
         end
-        raise KeepLocked.new res 
       end
+      res.abort_callback = Proc.new do
+        begin
+          lockfile.unlock #if File.exists? lockfile.path and lockfile.locked?
+        rescue Exception
+          Log.medium "Lockfile exception: " << $!.message
+        end
+      end
+      raise KeepLocked.new res 
     end
 
     case res
@@ -356,24 +381,29 @@ module Persist
 
   def self.persist_file(path, type, persist_options, &block)
 
-    if is_persisted?(path, persist_options)
-      Log.low "Persist up-to-date: #{ path } - #{Misc.fingerprint persist_options}"
-      return path if persist_options[:no_load]
-      return load_file(path, type) 
-    else
+    begin
+      if is_persisted?(path, persist_options)
+        Log.low "Persist up-to-date: #{ path } - #{Misc.fingerprint persist_options}"
+        return path if persist_options[:no_load]
+        return load_file(path, type) 
+      else
+        Open.rm path if Open.exists? path
+      end
+    rescue Exception
       Open.rm path if Open.exists? path
     end
 
 
+    lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
     begin
-
-      lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
       Misc.lock lock_filename do |lockfile|
 
-        if is_persisted?(path, persist_options)
-          Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
-          return path if persist_options[:no_load]
-          return load_file(path, type) 
+        Misc.insist do
+          if is_persisted?(path, persist_options)
+            Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
+            return path if persist_options[:no_load]
+            return load_file(path, type) 
+          end
         end
 
         Log.medium "Persist create: #{ path } - #{type} #{Misc.fingerprint persist_options}"
@@ -387,8 +417,12 @@ module Persist
         persist_options[:no_load] ? path : res
       end
 
-    rescue
+    rescue Lockfile::StolenLockError
+      Log.medium "Lockfile stolen: #{path}"
+      retry
+    rescue Exception
       Log.medium "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
+      Log.exception $!
       FileUtils.rm path if Open.exists? path 
       raise $!
     end

data/lib/rbbt/tsv.rb

@@ -62,7 +62,7 @@ module TSV
 
     data = nil
 
-    lock_filename = filename.nil? ? nil : Persist.persistence_path(filename, {:dir => TSV.lock_dir})
+    lock_filename = filename.nil? ? nil : Persist.persistence_path(filename + '.open', {:dir => TSV.lock_dir})
     Misc.lock lock_filename  do
       data = Persist.persist_tsv source, filename, options, persist_options do |data|
         if serializer

data/lib/rbbt/util/concurrency/processes.rb

@@ -61,7 +61,12 @@ class RbbtProcessQueue
       rescue Aborted
         Log.warn "Aborting process monitor"
         @processes.each{|p| p.abort }
-        @processes.each{|p| p.join }
+        @processes.each{|p| 
+          begin
+            p.join 
+          rescue ProcessFailed
+          end
+        }
       rescue Exception
         Log.warn "Process monitor exception: #{$!.message}"
         @processes.each{|p| p.abort }
@@ -114,7 +119,10 @@ class RbbtProcessQueue
       (@process_monitor.raise(Aborted.new); @process_monitor.join) if @process_monitor and @process_monitor.alive?
       (@callback_thread.raise(Aborted.new); @callback_thread.join) if @callback_thread and @callback_thread.alive?
     ensure
-      join
+      begin
+        join
+      rescue ProcessFailed
+      end
     end
   end
 

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -33,8 +33,10 @@ class RbbtProcessQueue
         rescue ClosedStream
         rescue Aborted, Interrupt
           Log.warn "Worker #{Process.pid} aborted"
-          Kernel.exit! -1
+          Kernel.exit! 0
+          #Kernel.exit! -1
         rescue Exception
+          Log.exception $!
           @callback_queue.push($!) if @callback_queue
           Kernel.exit! -1
         ensure
@@ -44,7 +46,8 @@ class RbbtProcessQueue
     end
 
     def join
-      joined_pid = Process.waitpid @pid
+      Process.waitpid @pid
+      raise ProcessFailed unless $?.success?
     end
 
     def abort

data/lib/rbbt/util/misc/development.rb

@@ -125,6 +125,12 @@ module Misc
     try = 0
     begin
       yield
+    rescue Aborted
+      if msg
+        Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
+      else
+        Log.warn("Not Insisting after Aborted: #{$!.message}")
+      end
     rescue Exception
       if msg
         Log.warn("Insisting after exception: #{$!.message} -- #{msg}")

data/lib/rbbt/util/misc/lock.rb

@@ -1,12 +1,16 @@
 if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
   Lockfile.refresh = false 
   Lockfile.max_age = 60 * 60 * 5
+  Lockfile.max_age = 15
+  Lockfile.suspend = 10
 else
+  Lockfile.dont_use_lock_id = true
   Lockfile.refresh = 5 
-  Lockfile.max_age = 5 * 60
-  Lockfile.suspend = 3
+  Lockfile.max_age = 60
+  Lockfile.suspend = 15
 end
 
+
 module Misc
 
   LOCK_MUTEX = Mutex.new
@@ -19,40 +23,16 @@ module Misc
     lock_path = File.expand_path(file + '.lock')
     lockfile = Lockfile.new(lock_path, options)
 
-    hostname = Misc.hostname
-    LOCK_MUTEX.synchronize do
-      Misc.insist 2, 0.1 do
-        Misc.insist 3, 0.1 do
-          begin
-            if File.exists? lock_path
-              info = Open.open(lock_path){|f| YAML.load(f) }
-              raise "No info" unless info
-
-              if hostname == info["host"] and not Misc.pid_exists?(info["pid"])
-                Log.high("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
-
-                FileUtils.rm lock_path
-              end
-            end
-          rescue Exception
-            FileUtils.rm lock_path if File.exists? lock_path
-            lockfile = Lockfile.new(lock_path, options) unless File.exists? lock_path
-            raise $!
-          end
-        end
-      end
-      lockfile.lock 
-    end
+    lockfile.lock 
 
     begin
       res = yield lockfile
-    rescue Lockfile::StolenLockError
-      unlock = false
+    #rescue Lockfile::StolenLockError
     rescue KeepLocked
       unlock = false
       res = $!.payload
     rescue Exception
-      lockfile.unlock if lockfile.locked?
+      lockfile.unlock #if lockfile.locked?
       raise $!
     ensure
       if unlock 

data/lib/rbbt/util/misc/pipes.rb

@@ -201,6 +201,7 @@ module Misc
     return if Open.exists? path
     tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
     Misc.lock tmp_path do
+      return if Open.exists? path
       if not Open.exists? path
         FileUtils.rm_f tmp_path if File.exists? tmp_path
         begin

data/lib/rbbt/util/open.rb

@@ -377,12 +377,13 @@ module Open
     if block_given?
       begin
         return yield(io)
-      rescue
+      rescue Exception
         io.abort if io.respond_to? :abort
         io.join if io.respond_to? :join
         raise $!
       ensure
         io.join if io.respond_to? :join
+        io.close if io.respond_to? :close and not io.closed?
       end
     else
       io

data/lib/rbbt/workflow/accessor.rb

@@ -5,6 +5,10 @@ class Step
    
   INFO_SERIALIAZER = Marshal
 
+  def self.started?
+    info_file.exists?
+  end
+  
   def self.wait_for_jobs(jobs)
     begin
       threads = []
@@ -465,7 +469,7 @@ module Workflow
 
   TAG = :hash
   def step_path(taskname, jobname, inputs, dependencies, extension = nil)
-    Proc.new{
+    #Proc.new{
       raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
       if inputs.any? or dependencies.any?
         tagged_jobname = case TAG
@@ -486,7 +490,7 @@ module Workflow
       end
 
       workdir[taskname][tagged_jobname].find
-    }
+    #}
   end
 
   def id_for(path)

data/lib/rbbt/workflow/step.rb

@@ -140,7 +140,7 @@ class Step
       @result = nil
       @pid = nil
 
-      begin
+      Misc.insist do
         Open.rm info_file if Open.exists? info_file
         Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
         Open.rm path if Open.exists? path

data/lib/rbbt/workflow/step/run.rb

@@ -143,8 +143,8 @@ class Step
           dependency.clean if (dependency.error? or dependency.aborted? or dependency.status.nil?)# or not dependency.done?)
         end
 
-        unless dependency.result or dependency.done?
-          dependency.run(true).grace
+        unless dependency.started? #dependency.result or dependency.done?
+          dependency.run(true)#.grace
         end
       rescue Aborted
         Log.error "Aborted dep. #{Log.color :red, dependency.task.name.to_s}"

data/share/rbbt_commands/system/clean

@@ -18,8 +18,6 @@ EOF
 rbbt_usage and exit 0 if options[:help]
 
 locks = Rbbt.locks
-sensiblewrites = Rbbt.sensiblewrites
-persists = Rbbt.persists
 
 puts Log.color(:magenta, "# System clean")
 puts
@@ -27,14 +25,15 @@ if locks.any?
   puts Log.color(:magenta, "Locks: #{locks.length}")
   locks.each do |lock|
     pid, ppid, time = Rbbt.load_lock(lock)
-    if not Misc.pid_exists? pid
+    if File.exists?(lock) and not Misc.pid_exists? pid
       puts "  Removing #{ lock }"
-      FileUtils.rm lock if File.exists? lock
+      FileUtils.rm lock #if File.exists? lock
     end
   end
   puts
 end
 
+persists = Rbbt.persists
 if persists.any?
   puts Log.color(:magenta, "Persist: #{persists.length}")
   persists.each do |persist|
@@ -49,6 +48,7 @@ if persists.any?
   puts
 end
 
+sensiblewrites = Rbbt.sensiblewrites
 if sensiblewrites.any?
   puts Log.color(:magenta, "Writing: #{sensiblewrites.length}")
   sensiblewrites.each do |sensiblewrite|
@@ -65,7 +65,7 @@ end
 
 jobs = Rbbt.jobs
 
-puts Log.color(:magenta, "# Workflows:")
+puts Log.color(:magenta, "# Workflow clean")
 puts
 jobs.each do |workflow, tasks|
   tasks.each do |task, jobs|

data/share/rbbt_commands/system/deleted_files

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/monitor'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Find process with deleted open files
+
+$ rbbt system deleted_files 
+
+-h--help Print this help
+EOF
+rbbt_usage and exit 0 if options[:help]
+
+pids = {}
+Dir.glob('/proc/*/fd/*').each do |file|
+  pid = file.split('/')[-3]
+  begin
+    dest = File.readlink(file)
+  rescue
+    next
+  end
+  next if File.exists? dest
+  pids[pid] ||= []
+  pids[pid] << dest
+end
+
+pids.sort_by{|p,files| files.length}.each do |pid,files|
+    dirs = {}
+    files.each do |f|
+        name = File.basename(f)
+        dir = File.basename(File.dirname(f))
+        next if dir =~ /^pts|\.$/
+        dirs[dir] ||= []
+        dirs[dir] << name
+    end
+    next if dirs.empty?
+    puts Log.color :magenta, pid
+    dirs.sort_by{|dir,list| list.length}.each do |dir,list|
+        puts Misc.format_definition_list_item(dir, list.length.to_s)
+    end
+end
+
+puts "Total: #{pids.values.collect{|k,v| v}.flatten.length}"

data/share/rbbt_commands/system/status

@@ -13,12 +13,12 @@ Report the status of the system
 
 $ rbbt system status 
 
--u--uncomplete Print only uncompleted or error jobs
+-a--all Print all jobs, not only uncompleted
 -h--help Print this help
 EOF
 rbbt_usage and exit 0 if options[:help]
 
-uncomplete = options.delete :uncomplete
+all = options.delete :all
 
 def pid_msg(pid)
   color = if pid and Misc.pid_exists? pid
@@ -51,12 +51,10 @@ def status_msg(status)
   Log.color(color, status.to_s)
 end
 
-locks = Rbbt.locks
-sensiblewrites = Rbbt.sensiblewrites
-persists = Rbbt.persists
-
 puts Log.color(:magenta, "# System report")
 puts
+
+locks = Rbbt.locks
 if locks.any?
   puts Log.color(:magenta, "Locks:")
   locks.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |lock|
@@ -68,6 +66,7 @@ if locks.any?
   puts
 end
 
+persists = Rbbt.persists
 if persists.any?
   puts Log.color(:magenta, "Persist:")
   persists.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |persist|
@@ -78,6 +77,7 @@ if persists.any?
   puts
 end
 
+sensiblewrites = Rbbt.sensiblewrites
 if sensiblewrites.any?
   puts Log.color(:magenta, "Writing:")
   sensiblewrites.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |sensiblewrite|
@@ -89,10 +89,10 @@ if sensiblewrites.any?
   puts
 end
 
-jobs = Rbbt.jobs
 
 puts Log.color(:magenta, "# Workflows")
-puts
+
+jobs = Rbbt.jobs
 jobs.each do |workflow, tasks|
   tasks.each do |task, jobs|
     done = []
@@ -106,7 +106,7 @@ jobs.each do |workflow, tasks|
         other[status||"missing"] << [file, h[:pid]]
       end
     end
-    next if uncomplete and other.empty?
+    next if not all and other.empty?
     puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:cyan, task) << ": " <<  Log.color(:blue, done.length.to_s) << " done"
     other.each do |status, list|
       files_txt = list.collect{|f,p| p.nil? ? f : (f + " (#{pid_msg p})") }

data/test/rbbt/util/misc/test_lock.rb

@@ -2,30 +2,77 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_he
 
 require 'rbbt-util'
 require 'rbbt/util/misc/lock'
+require 'rbbt/workflow'
 
 class TestLock < Test::Unit::TestCase
   def __test_stress
-    size = 1000000
-    num = 50
-    cpus = 200
-    TmpFile.with_file do |dir|
-      TSV.traverse (0..size).to_a, :cpus => cpus, :type => :array, :bar => true do |i|
-        begin
-          v = rand(num)
-          file = File.join(dir, "file-" << v.to_s)
-          Persist.persist("foo", :string, :file => file, :update => true) do
-            Process.pid.to_s
+  end
+end
+
+if __FILE__ == $0
+  def deleted(pid = Process.pid)
+    begin
+      txt = `ls -la /proc/#{pid}/fd |grep deleted`
+      puts Log.color(:magenta, [pid, txt.split("\n")*", "] * ": ")
+    rescue Exception
+      Log.exception $!
+    end
+  end
+
+  cpus = 10
+  file = "/tmp/test.lock"
+
+  pids = []
+  cpus.times do 
+    pids << Process.fork do
+      while true do
+        Lockfile.new file do
+          Lockfile.new file + '.1' do
           end
+          File.open(file){|f| puts f.read }
+        end
+        deleted
+      end
+    end
+  end
+
+
+
+  exit
+  size = 1000000
+  num = 1
+  cpus = 5
 
-          txt = `ls -la /proc/#{Process.pid}/fd |grep deleted`
-          Open.write(file, txt)
-          puts [Process.pid, txt.split("\n").length] * ": "
-        rescue Exception
-          Log.exception $!
-          raise $!
+  pdb = "http://interactome3d.irbbarcelona.org/pdb.php?dataset=human&type1=interactions&type2=pdb&pdb=Q99685-Q99685-EXP-3hju.pdb1-B-0-A-0.pdb"
+  Workflow.require_workflow "Structure"
+  TmpFile.with_file do |dir|
+    Structure.workdir = dir
+    Path.setup dir
+    Log.severity = 4
+    TSV.traverse (0..size).to_a, :cpus => cpus, :type => :array, :bar => true do |i|
+      begin
+        v = rand(num).to_s
+        file = File.join(dir, "file-" << v.to_s)
+
+        Misc.lock file + '.produce' do
+          Misc.lock file  do
+          ##job = Structure.job(:neighbour_map, v, :pdb => pdb)
+          #job = Translation.example_step(:translate, "Example")
+          #job.path = file
+          #if job.done?
+          #  job.clean if rand < 0.3
+          #else
+          #  job.run(true) 
+          #end
+          end
         end
+        deleted Process.pid
+
+      rescue Exception
+        raise $!
       end
     end
   end
 end
 
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.30
+  version: 5.13.31
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-01 00:00:00.000000000 Z
+date: 2014-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -247,6 +247,7 @@ files:
 - share/rbbt_commands/stat/log
 - share/rbbt_commands/study/task
 - share/rbbt_commands/system/clean
+- share/rbbt_commands/system/deleted_files
 - share/rbbt_commands/system/purge
 - share/rbbt_commands/system/report
 - share/rbbt_commands/system/status