rbbt-util 5.13.31 → 5.13.32

data/lib/rbbt/workflow/step.rb

@@ -130,7 +130,9 @@ class Step
     exec
   end
 
-  def clean
+  def self.clean(path)
+    info_file = Step.info_file path
+    files_dir = Step.files_dir path
     if Open.exists?(path) or Open.exists?(info_file)
       begin
         self.abort if self.running?
@@ -148,6 +150,10 @@ class Step
         Open.rm_rf files_dir if Open.exists? files_dir
       end
     end
+  end
+
+  def clean
+    Step.clean(path)
     self
   end
 

data/lib/rbbt/workflow.rb

@@ -233,22 +233,28 @@ module Workflow
     end 
   end
 
+  def step_module
+    @_m ||= begin
+              m = Module.new
+
+              helpers.each do |name,block|
+                m.class.send(:define_method, name, &block)
+              end
+
+              m
+            end
+    @_m
+  end
+
   def get_job_step(step_path, task = nil, input_values = nil, dependencies = nil)
     step_path = step_path.call if Proc === step_path
     persist = input_values.nil? ? false : true
     persist = false
     key = Path === step_path ? step_path.find : step_path
-    step = Persist.memory("Step", :key => key, :repo => step_cache, :persist => persist) do
     step = Step.new step_path, task, input_values, dependencies
 
-      helpers.each do |name, block|
-        (class << step; self; end).instance_eval do
-          define_method name, &block
-        end
-      end
 
-      step
-    end
+    step.extend step_module
 
     step.task ||= task
     step.inputs ||= input_values
@@ -268,12 +274,13 @@ module Workflow
 
     Workflow.resolve_locals(inputs)
 
-    dependencies = real_dependencies(task, jobname, inputs, task_dependencies[taskname] || [])
-
-    real_inputs = {}
     task_inputs = task_info(taskname)[:inputs]
     defaults = task_info(taskname)[:input_defaults]
 
+    dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
+
+    real_inputs = {}
+
     inputs.each do |k,v|
       real_inputs[k] = v if (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s)) and (defaults[k].to_s != v.to_s and not (FalseClass === v and defaults[k].nil?))
     end

data/share/rbbt_commands/system/clean

@@ -9,88 +9,87 @@ $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_c
 
 options = SOPT.setup <<EOF
 
-Clean failed jobs, lock files, etc
+Clean orphaned files
 
-$ rbbt system clean 
+$ rbbt system clean <workflow> <task>
 
+-a--all Print all jobs, not only uncompleted
+-f--force Remove regardless of been active
 -h--help Print this help
 EOF
 rbbt_usage and exit 0 if options[:help]
 
-locks = Rbbt.locks
+workflow, task = ARGV
+
+workflow = workflow.split "," if workflow
+task = task.split "," if task
+
+all = options.delete :all
+force = options.delete :force
 
 puts Log.color(:magenta, "# System clean")
-puts
+
+locks = Rbbt.lock_info
 if locks.any?
-  puts Log.color(:magenta, "Locks: #{locks.length}")
-  locks.each do |lock|
-    pid, ppid, time = Rbbt.load_lock(lock)
-    if File.exists?(lock) and not Misc.pid_exists? pid
-      puts "  Removing #{ lock }"
-      FileUtils.rm lock #if File.exists? lock
+  puts
+  puts Log.color(:magenta, "Locks:")
+  locks.each do |file,info|
+    if force or (info[:pid] and Misc.pid_exists? info[:pid])
+      puts "  Removing #{ file }"
+      File.unlink file 
     end
   end
-  puts
 end
 
-persists = Rbbt.persists
+persists = Rbbt.persist_info
 if persists.any?
-  puts Log.color(:magenta, "Persist: #{persists.length}")
-  persists.each do |persist|
-    lf = persist + '.lock'
-    pid, ppid, time = Rbbt.load_lock(lf)
-    if not Misc.pid_exists? pid
-      puts "  Removing #{ persist }"
-      FileUtils.rm persists if File.exists? persists
-      FileUtils.rm lf if File.exists? lf
+  puts
+  puts Log.color(:magenta, "Persist:")
+  persists.each do |file,info|
+    if force or (info[:pid] and Misc.pid_exists? info[:pid])
+      puts "  Removing #{ file }"
+      File.unlink file 
     end
   end
-  puts
 end
 
-sensiblewrites = Rbbt.sensiblewrites
+sensiblewrites = Rbbt.sensiblewrite_info
 if sensiblewrites.any?
-  puts Log.color(:magenta, "Writing: #{sensiblewrites.length}")
-  sensiblewrites.each do |sensiblewrite|
-    lf = sensiblewrite + '.lock'
-    pid, ppid, time = Rbbt.load_lock(lf)
-    if not Misc.pid_exists? pid
-      puts "  Removing #{ sensiblewrite }"
-      FileUtils.rm sensiblewrite  if File.exists? sensiblewrite
-      FileUtils.rm lf if File.exists? lf
+  puts
+  puts Log.color(:magenta, "Writes:")
+  sensiblewrites.each do |file,info|
+    if force or (info[:pid] and Misc.pid_exists? info[:pid])
+      puts "  Removing #{ file }"
+      File.unlink file 
     end
   end
-  puts
 end
 
-jobs = Rbbt.jobs
-
+puts
 puts Log.color(:magenta, "# Workflow clean")
 puts
-jobs.each do |workflow, tasks|
-  tasks.each do |task, jobs|
-    done = []
-    other = {}
-    jobs.each do |file,h|
-      #puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": #{jobs.length}"
-      status = h[:status]
-      if h[:status] == :done
-        done << file
-      else
-        other[status||"missing"] ||= []
-        other[status||"missing"] << [file, h[:pid]]
-      end
-    end
-    other.each do |status, list|
-      list.each do |f,p|
-        if not Misc.pid_exists?(p) or status.to_s =~ /error|aborted|missing/
-          puts "  Removing #{ f }"
-          FileUtils.rm_rf f if File.exists? f
-          FileUtils.rm_rf f + '.info' if File.exists? f + '.info'
-          FileUtils.rm_rf f + '.files' if File.exists? f + '.files'
-        end
-      end
-    end
-  end
+
+jobs = Rbbt.job_info workflow, task
+workflows = {}
+
+jobs.each do |file,i|
+  next if i[:done]
+  info = begin
+           Open.open(i[:info_file]) do |f|
+             Marshal.load(f)
+           end
+         rescue
+           {:status => :noinfo}
+         end
+
+  pid = info[:pid]
+
+  status = info[:status]
+  status = :missing if status == :done and not File.exists? file
+  status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
+
+  status = status.to_s
+  puts "  Removing #{ file } - #{status}" 
+  Step.clean(file) if status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status == ""
 end
 

data/share/rbbt_commands/system/status

@@ -11,13 +11,18 @@ options = SOPT.setup <<EOF
 
 Report the status of the system
 
-$ rbbt system status 
+$ rbbt system status <workflow> <task>
 
 -a--all Print all jobs, not only uncompleted
 -h--help Print this help
 EOF
 rbbt_usage and exit 0 if options[:help]
 
+workflow, task = ARGV
+
+workflow = workflow.split "," if workflow
+task = task.split "," if task
+
 all = options.delete :all
 
 def pid_msg(pid)
@@ -27,15 +32,15 @@ def pid_msg(pid)
             :red
           end
   if pid.nil?
-    Log.color(:yellow, "missing")
+    ""
   else
     Log.color(color, pid)
   end
 end
 
 def status_msg(status)
-  color = case status
-          when :error, :aborted, :missing
+  color = case status.to_sym
+          when :error, :aborted, :missing, :dead
             :red
           when :streaming, :started
             :yellow
@@ -54,37 +59,33 @@ end
 puts Log.color(:magenta, "# System report")
 puts
 
-locks = Rbbt.locks
+locks = Rbbt.lock_info
 if locks.any?
   puts Log.color(:magenta, "Locks:")
-  locks.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |lock|
-    next unless File.exists?(lock)
-    pid, ppid, time = Rbbt.load_lock(lock)
-    time = File.exists?(lock) ? File.ctime(lock) : Time.now
-    puts "  " << lock + Log.color(:blue, " -- time: #{Misc.format_seconds(Time.now - time)}; ppid: #{ppid}; pid: #{pid_msg pid}")
+  locks.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed, pid, ppid = info.values_at :elapsed, :pid, :ppid
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}; ppid: #{ppid}; pid: #{pid_msg pid}")
   end
   puts
 end
 
-persists = Rbbt.persists
+
+persists = Rbbt.persist_info
 if persists.any?
   puts Log.color(:magenta, "Persist:")
-  persists.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |persist|
-    next unless File.exists?(persist)
-    time = File.exists?(persist) ? File.ctime(persist) : Time.now
-    puts "  " << persist + Log.color(:blue, " -- time: #{Misc.format_seconds(Time.now - time)})")
+  persists.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
   end
   puts
 end
 
-sensiblewrites = Rbbt.sensiblewrites
+sensiblewrites = Rbbt.sensiblewrite_info
 if sensiblewrites.any?
   puts Log.color(:magenta, "Writing:")
-  sensiblewrites.sort_by{|f| File.exists?(f) ? File.ctime(f) : Time.now}.each do |sensiblewrite|
-    next unless File.exists?(sensiblewrite)
-    pid, ppid, time = Rbbt.load_lock(sensiblewrite + '.lock')
-    time = File.exists?(sensiblewrite) ? File.ctime(sensiblewrite) : Time.now
-    puts "  " << sensiblewrite + Log.color(:blue, " -- time: #{Misc.format_seconds(Time.now - time)}; ppid: #{ppid}; pid: #{pid_msg pid}")
+  sensiblewrites.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
   end
   puts
 end
@@ -92,26 +93,44 @@ end
 
 puts Log.color(:magenta, "# Workflows")
 
-jobs = Rbbt.jobs
-jobs.each do |workflow, tasks|
-  tasks.each do |task, jobs|
-    done = []
-    other = {}
-    jobs.each do |file,h|
-      status = h[:status]
-      if h[:status] == :done
-        done << file
-      else
-        other[status||"missing"] ||= []
-        other[status||"missing"] << [file, h[:pid]]
-      end
+jobs = Rbbt.job_info workflow, task
+workflows = {}
+
+jobs.each do |file,info|
+  next unless all or not info[:done]
+  workflow = info[:workflow]
+  task = info[:task]
+  workflows[workflow] ||= {}
+  workflows[workflow][task] ||= {}
+  workflows[workflow][task][file] ||= info
+end
+
+workflows.sort.each do |workflow,tasks|
+  tasks.sort.each do |task,jobs|
+    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": " <<  Log.color(:blue, jobs.length.to_s)
+
+    files_txt = jobs.collect do |file, i|
+      info = begin
+               Open.open(i[:info_file]) do |f|
+                 Marshal.load(f)
+               end
+             rescue
+               Log.exception $!
+               {:status => :noinfo}
+             end
+
+      pid = info[:pid]
+      status = info[:status]
+      status = :missing if status == :done and not File.exists? file
+      status = status.to_s
+      status << Log.color(:red, " (dead)") if status != "done" and pid and not Misc.pid_exists?(pid)
+      str = file.dup
+      str << " #{ status_msg status }"
+      str << "; #{pid_msg pid}" unless status == "done"
+      str
     end
-    next if not all and other.empty?
-    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:cyan, task) << ": " <<  Log.color(:blue, done.length.to_s) << " done"
-    other.each do |status, list|
-      files_txt = list.collect{|f,p| p.nil? ? f : (f + " (#{pid_msg p})") }
-      puts "  " << status_msg(status) << ": " << (files_txt * ", ")
+    files_txt.each do |f|
+      puts "  " << f
     end
   end
 end
-

data/share/rbbt_commands/tsv/info

@@ -53,7 +53,7 @@ else
       puts "  - #{Log.color :cyan, i + 1}: " << Log.color(:yellow, f)
     end
   end
-  puts "Rows: #{Log.color :blue, `wc -l #{ file }|cut -f 1 -d' '`}" unless Open.remote? file
+  puts "Rows: #{Log.color :blue, `wc -l '#{ file }'|cut -f 1 -d' '`}" unless Open.remote? file
   parts = []
   header.first_line.split(header.sep).each_with_index{|p,i| parts << (Log.color(:cyan, "(#{i}) ") << p.strip) }
   puts parts * "\t"

data/test/rbbt/test_monitor.rb

@@ -0,0 +1,13 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'test_helper.rb')
+require 'rbbt/monitor'
+
+class TestMonitor < Test::Unit::TestCase
+  def _test_locks
+  end
+end
+
+if __FILE__ == $0
+  jobs = Rbbt.jobs("Structure", "annotate")
+
+end
+

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -6,7 +6,7 @@ class StopException < StandardError; end
 
 class TestTSVParallelThrough < Test::Unit::TestCase
 
-  def test_traverse_tsv
+  def _test_traverse_tsv
     require 'rbbt/sources/organism'
 
     head = 100
@@ -25,7 +25,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def test_traverse_tsv_cpus
+  def _test_traverse_tsv_cpus
     require 'rbbt/sources/organism'
 
     head = 100
@@ -47,7 +47,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert res.values.compact.flatten.uniq.length > 0
   end
 
-  def test_traverse_stream
+  def _test_traverse_stream
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -61,7 +61,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def test_traverse_stream_cpus
+  def _test_traverse_stream_cpus
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -75,7 +75,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.keys.compact.sort.length
   end
 
-  def test_traverse_stream_keys
+  def _test_traverse_stream_keys
     require 'rbbt/sources/organism'
 
     head = 1000
@@ -99,7 +99,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal res.sort, Organism.identifiers("Hsa").tsv(:head => head).keys.sort
   end
   
-  def test_traverse_array
+  def _test_traverse_array
     require 'rbbt/sources/organism'
 
     array = []
@@ -121,7 +121,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array, res
   end
 
-  def test_traverse_array_threads
+  def _test_traverse_array_threads
     require 'rbbt/sources/organism'
 
     array = []
@@ -142,7 +142,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array.sort, res.sort
   end
 
-  def test_traverse_array_cpus
+  def _test_traverse_array_cpus
     require 'rbbt/sources/organism'
 
     array = []
@@ -157,7 +157,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal array.sort, res.sort
   end
 
-  def test_traverse_benchmark
+  def _test_traverse_benchmark
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -179,7 +179,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     end
   end
 
-  def test_traverse_into_dumper
+  def _test_traverse_into_dumper
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -197,7 +197,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def test_traverse_into_dumper_threads
+  def _test_traverse_into_dumper_threads
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -217,7 +217,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def test_traverse_into_dumper_cpus
+  def _test_traverse_into_dumper_cpus
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -238,7 +238,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
 
   #{{{ TRAVERSE DUMPER
 
-  def test_traverse_dumper
+  def _test_traverse_dumper
     require 'rbbt/sources/organism'
 
     head = 2_000
@@ -259,14 +259,16 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def test_traverse_dumper_threads
+  def _test_traverse_dumper_threads
     require 'rbbt/sources/organism'
 
     head = 2_000
     threads = 3
 
     tsv = TSV::Parser.new Organism.identifiers("Hsa").open, :head => head
+
     dumper = TSV::Dumper.new tsv.options
+    dumper.init
 
     TSV.traverse tsv, :head => head, :threads => threads, :into => dumper do |k,v|
       k = k.first
@@ -281,7 +283,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
-  def test_traverse_dumper_cpus
+  def _test_traverse_dumper_cpus
     require 'rbbt/sources/organism'
 
     head = 10_000
@@ -303,6 +305,26 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
+  def _test_traverse_dumper_exception
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+
+    Log.info Log.color :red, "TRAVERSE EXCEPTION"
+    stream = Organism.identifiers("Hsa/jun2011").open 
+    dumper = TSV::Dumper.new Organism.identifiers("Hsa/jun2011").tsv_options
+    dumper.init
+
+    assert_raise StopException do
+      TSV.traverse stream, :head => head, :into => dumper do |k,v|
+        k = k.first
+        raise StopException if rand(100) < 20
+        [k,v]
+      end
+      dumper.stream.join
+    end
+  end
+
   def test_traverse_dumper_cpus_exception
     require 'rbbt/sources/organism'
 
@@ -314,17 +336,17 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     dumper = TSV::Dumper.new Organism.identifiers("Hsa/jun2011").tsv_options
     dumper.init
 
-    assert_raise StopException do
-      TSV.traverse stream, :head => head, :_cpus => cpus, :into => dumper do |k,v|
+    assert_raise ProcessFailed do
+      TSV.traverse stream, :head => head, :cpus => cpus, :into => dumper do |k,v|
         k = k.first
-        raise StopException if rand(100) < 1
+        raise StopException if rand(100) < 20
         [k,v]
       end
-      Misc.consume_stream dumper.stream 
+      dumper.stream.join
     end
   end
 
-  def test_traverse_into_stream
+  def _test_traverse_into_stream
     size = 100
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :into => :stream do |e|
@@ -333,7 +355,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal size, stream.read.split("\n").length
   end
 
-  def test_traverse_progress
+  def _test_traverse_progress
     size = 1000
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
@@ -367,7 +389,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal size, stream.read.split("\n").length
   end
 
-  def test_store_multiple
+  def _test_store_multiple
     size = 1000
     array = (1..size).to_a.collect{|n| n.to_s}
     stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|

data/test/rbbt/util/test_misc.rb

@@ -6,6 +6,16 @@ require 'rbbt/entity'
 
 class TestMisc < Test::Unit::TestCase
 
+  def test_object_delta
+    a = []
+    b = nil
+    d = Misc.object_delta(String) do
+      a << rand.to_s
+    end
+    assert_equal 1, a.length
+    assert_match /^0\.\d+$/, a.first
+  end
+
   def test_format_seconds
     t = 61.3232
     assert_equal "00:01:01", Misc.format_seconds(t)

data/test/rbbt/util/test_open.rb

@@ -132,5 +132,21 @@ class TestOpen < Test::Unit::TestCase
     end
   end
   
+  def test_repo_dir
+    TmpFile.with_file do |tmpdir|
+      tmpdir = "/home/mvazquezg/tmp/repo_dir"
+      repo = File.join(tmpdir, 'repo')
+
+      Open.repository_dirs.push(repo)
+
+      obj = { :a => "???a"}
+      filename = "file" << (rand * 100).to_i.to_s
+      Open.write(File.join(repo, filename), Marshal.dump(obj))
+      dump = Open.read(File.join(repo, filename))
+      obj_cp = Marshal.load(dump)
+      assert_equal obj, obj_cp
+    end
+  end
+  
 end