rbbt-util 5.13.37 → 5.14.0

data/lib/rbbt/workflow.rb

@@ -275,14 +275,18 @@ module Workflow
     Workflow.resolve_locals(inputs)
 
     task_inputs = task_info(taskname)[:inputs]
-    defaults = task_info(taskname)[:input_defaults]
+    defaults = IndiferentHash.setup(task_info(taskname)[:input_defaults])
 
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
 
     real_inputs = {}
 
     inputs.each do |k,v|
-      real_inputs[k] = v if (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s)) and (defaults[k].to_s != v.to_s and not (FalseClass === v and defaults[k].nil?))
+      default = defaults[k]
+      if (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s)) and 
+        (defaults[k].to_s != v.to_s and not (FalseClass === v and defaults[k].nil?))
+        real_inputs[k] = v 
+      end
     end
 
     if real_inputs.empty?

data/lib/rbbt/workflow/accessor.rb

@@ -2,7 +2,8 @@ require 'rbbt/util/open'
 require 'yaml'
 
 class Step
-   
+
+
   INFO_SERIALIAZER = Marshal
 
   def self.started?
@@ -12,7 +13,7 @@ class Step
   def self.wait_for_jobs(jobs)
     begin
       threads = []
-      jobs.each do |j| threads << Thread.new{j.grace.join} end
+      jobs.each do |j| threads << Thread.new{j.join} end
       threads.each{|t| t.join }
     rescue Exception
       threads.each{|t| t.exit }
@@ -66,33 +67,40 @@ class Step
     @info_file ||= Step.info_file(path)
   end
 
-  def info
+  def info_lock
+    @info_lock ||= begin
+                     path = Persist.persistence_path(info_file + '.lock', {:dir => Step.lock_dir})
+                     Lockfile.new path
+                   end
+  end
+
+  def info(check_lock = true)
     return {} if info_file.nil? or not Open.exists? info_file
     begin
-      @info_mutex.synchronize do
-        begin
-          return @info_cache if @info_cache and File.mtime(info_file) < @info_cache_time
-        rescue Exception
-          raise $!
-        end
+      begin
+        return @info_cache if @info_cache and File.ctime(info_file) < @info_cache_time
+      rescue Exception
+        raise $!
+      end
 
-        begin
-          @info_cache = Misc.insist(2, 3, info_file) do
-            Misc.insist(2, 1, info_file) do
-              Misc.insist(3, 0.2, info_file) do
-                Open.open(info_file) do |file|
-                  INFO_SERIALIAZER.load(file) || {}
-                end
+      begin
+        @info_cache = Misc.insist(2, 3, info_file) do
+          Misc.insist(2, 1, info_file) do
+            Misc.insist(3, 0.2, info_file) do
+              raise TryAgain, "Info locked" if check_lock and info_lock.locked?
+              Open.open(info_file) do |file|
+                INFO_SERIALIAZER.load(file) #|| {}
               end
             end
           end
-          @info_cache_time = Time.now
-          @info_cache
         end
+        @info_cache_time = Time.now
+        @info_cache
       end
     rescue Exception
       Log.debug{"Error loading info file: " + info_file}
-      Open.write(info_file, INFO_SERIALIAZER.dump({:status => :error, :messages => ["Info file lost"]}))
+      Log.exception $!
+      Open.sensiblewrite(info_file, INFO_SERIALIAZER.dump({:status => :error, :messages => ["Info file lost"]}))
       raise $!
     end
   end
@@ -100,12 +108,11 @@ class Step
   def set_info(key, value)
     return nil if @exec or info_file.nil?
     value = Annotated.purge value if defined? Annotated
-    lock_filename = Persist.persistence_path(info_file, {:dir => Step.lock_dir})
-    Open.lock(info_file, :refresh => false) do
-      i = info
+    Open.lock(info_file, :lock => info_lock) do
+      i = info(false)
       i[key] = value 
       @info_cache = i
-      Open.write(info_file, INFO_SERIALIAZER.dump(i))
+      Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true)
       @info_cache_time = Time.now
       value
     end
@@ -114,12 +121,11 @@ class Step
   def merge_info(hash)
     return nil if @exec or info_file.nil?
     value = Annotated.purge value if defined? Annotated
-    lock_filename = Persist.persistence_path(info_file, {:dir => Step.lock_dir})
-    Open.lock(info_file, :refresh => false) do
-      i = info
+    Open.lock(info_file, :lock => info_lock) do
+      i = info(false)
       i.merge! hash
       @info_cache = i
-      Open.write(info_file, INFO_SERIALIAZER.dump(i))
+      Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true)
       @info_cache_time = Time.now
       value
     end
@@ -416,10 +422,11 @@ module Workflow
 
   def rec_dependencies(taskname)
     if task_dependencies.include? taskname
-      deps = task_dependencies[taskname].select{|dep| String === dep or Symbol === dep or Array === dep}
-      all_deps = deps.dup
+      deps = task_dependencies[taskname]
+      all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
       deps.each do |dep| 
-        if Array === dep 
+        case dep
+        when Array
           dep.first.rec_dependencies(dep.last).each do |d|
             if Array === d
               all_deps << d
@@ -427,8 +434,10 @@ module Workflow
               all_deps << [dep.first, d]
             end
           end
-        else
+        when String, Symbol
           all_deps.concat rec_dependencies(dep.to_sym)
+        when DependencyBlock
+          all_deps << dep.dependency
         end
       end
       all_deps.uniq
@@ -437,28 +446,49 @@ module Workflow
     end
   end
 
+  def task_from_dep(dep)
+    case dep
+    when Array
+      dep.first.tasks[dep.last] 
+    when String
+      tasks[dep.to_sym]
+    when Symbol
+      tasks[dep.to_sym]
+    else
+      raise "Unknown dependency: #{Misc.fingerprint dep}"
+    end
+  end
+
   def rec_inputs(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat((Array === tn ? tn.first.tasks[tn.last] : tasks[tn.to_sym]).inputs) }.uniq
+    [taskname].concat(rec_dependencies(taskname)).inject([]){|acc, tn| acc.concat(task_from_dep(tn).inputs) }.uniq
   end
 
   def rec_input_defaults(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge((Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_defaults) }.
-      tap{|h| IndiferentHash.setup(h)}
+    [taskname].concat(rec_dependencies(taskname)).inject(IndiferentHash.setup({})){|acc, tn|
+      new = (Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_defaults
+      acc = new.merge(acc) 
+    }.tap{|h| IndiferentHash.setup(h)}
   end
 
   def rec_input_types(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge((Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_types) }.
-      tap{|h| IndiferentHash.setup(h) }
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
+      new = (Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_types
+      acc = new.merge(acc) 
+    }.tap{|h| IndiferentHash.setup(h)}
   end
 
   def rec_input_descriptions(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge((Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_descriptions) }.
-      tap{|h| IndiferentHash.setup(h)}
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
+      new = (Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_descriptions
+      acc = new.merge(acc) 
+    }.tap{|h| IndiferentHash.setup(h)}
   end
 
   def rec_input_options(taskname)
-    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge((Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_options)}.
-      tap{|h| IndiferentHash.setup(h)}
+    [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
+      new = (Array === tn ? tn.first.tasks[tn.last.to_sym] : tasks[tn.to_sym]).input_options
+      acc = new.merge(acc) 
+    }.tap{|h| IndiferentHash.setup(h)}
   end
 
   def real_dependencies(task, jobname, inputs, dependencies)

data/lib/rbbt/workflow/definition.rb

@@ -4,6 +4,15 @@ require 'rbbt/workflow/annotate'
 module Workflow
   include AnnotatedModule
 
+  module DependencyBlock
+    attr_accessor :dependency
+    def self.setup(block, dependency)
+      block.extend DependencyBlock
+      block.dependency = dependency
+      block
+    end
+  end
+
   AnnotatedModule.add_consummable_annotation(self,
                                              :result_description => "",
                                              :result_type        => nil,
@@ -36,6 +45,20 @@ module Workflow
     end
   end
 
+  def dep(*dependency, &block)
+    @dependencies ||= []
+    if block_given?
+      DependencyBlock.setup block, dependency if dependency.any?
+      @dependencies << block
+    else
+      if Module === dependency.first
+        @dependencies << dependency
+      else
+        @dependencies.concat dependency
+      end
+    end
+  end
+
   def task(name, &block)
     if Hash === name
       type = name.first.last

data/lib/rbbt/workflow/step.rb

@@ -14,7 +14,11 @@ class Step
     attr_accessor :lock_dir
     
     def lock_dir
-      @lock_dir ||= Rbbt.tmp.step_info_locks.find
+      @lock_dir ||= begin
+                      dir = Rbbt.tmp.step_info_locks.find
+                      FileUtils.mkdir_p dir unless Open.exists? dir
+                      dir
+                    end
     end
   end
 
@@ -35,11 +39,18 @@ class Step
     @mutex = Mutex.new
     @info_mutex = Mutex.new
     @inputs = inputs || []
-    NamedArray.setup @inputs, task.inputs if task
+    NamedArray.setup @inputs, task.inputs.collect{|s| s.to_s} if task and task.inputs
   end
 
   def inputs
-    NamedArray.setup @inputs, task.inputs if task.inputs and not NamedArray === @inputs
+    if @inputs.nil? and task and task.respond_to? :inputs
+      @inputs = info[:inputs].values_at *task.inputs.collect{|name| name.to_s}
+    end
+
+    if task.inputs and not NamedArray === @inputs
+      NamedArray.setup @inputs, task.inputs 
+    end
+
     @inputs
   end
 
@@ -188,6 +199,7 @@ class Step
       else
       end
     end
+    self
   end
 
   def step(name)

data/lib/rbbt/workflow/step/run.rb

@@ -100,17 +100,21 @@ class Step
   end
 
   def kill_children
-    children_pids = info[:children_pids]
-    if children_pids and children_pids.any?
-      Log.medium("Killing children: #{ children_pids * ", " }")
-      children_pids.each do |pid|
-        Log.medium("Killing child #{ pid }")
-        begin
-          Process.kill "INT", pid
-        rescue Exception
-          Log.medium("Exception killing child #{ pid }: #{$!.message}")
+    begin
+      children_pids = info[:children_pids]
+      if children_pids and children_pids.any?
+        Log.medium("Killing children: #{ children_pids * ", " }")
+        children_pids.each do |pid|
+          Log.medium("Killing child #{ pid }")
+          begin
+            Process.kill "INT", pid
+          rescue Exception
+            Log.medium("Exception killing child #{ pid }: #{$!.message}")
+          end
         end
       end
+    rescue
+      Log.medium("Exception finding children")
     end
   end
 
@@ -276,9 +280,10 @@ class Step
       Misc.pre_fork
       begin
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
-        FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
+        FileUtils.mkdir_p File.dirname(path) unless File.exists? File.dirname(path)
         begin
           res = run true
+          set_info :forked, true
         rescue Aborted
           Log.debug{"Forked process aborted: #{path}"}
           log :aborted, "Job aborted (#{Process.pid})"
@@ -306,15 +311,15 @@ class Step
           end
         rescue Exception
           Log.debug("Exception waiting for children: #{$!.message}")
-          exit -1
+          RbbtSemaphore.post_semaphore(semaphore) if semaphore
+          Kernel.exit! -1
         end
         set_info :pid, nil
-        exit 0
       ensure
         RbbtSemaphore.post_semaphore(semaphore) if semaphore
       end
+      Kernel.exit! 0
     end
-    set_info :forked, true
     Process.detach(@pid)
     self
   end

data/lib/rbbt/workflow/usage.rb

@@ -27,7 +27,10 @@ module Task
     if deps and deps.any?
       puts Log.color(:magenta, "Inputs from dependencies:")
       puts
+      seen = []
       deps.each do |dep|
+        next if seen.include? dep.name
+        seen << dep.name
         puts "  #{Log.color :yellow, dep.name.to_s}:"
         puts
         puts SOPT.input_doc((dep.inputs - self.inputs), dep.input_types, dep.input_descriptions, dep.input_defaults, true)

data/share/Rlib/util.R

@@ -77,7 +77,7 @@ rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1, check
   data=read.table(file=filename, sep=sep, fill=fill, as.is=as.is, quote=quote, row.names= row.names, comment.char = comment.char, ...);
   f = file(filename, 'r');
   headers = readLines(f, 1);
-  if (length(grep("^#: ", headers)) > 0){ 
+  if (length(grep("^#:", headers)) > 0){ 
       headers = readLines(f, 1); 
   } 
   if (length(grep("^#", headers)) > 0){

data/share/rbbt_commands/tsv/get

@@ -17,12 +17,10 @@ else
   tsv = TSV.open(file, :grep => value )
 end
 
-
 res = tsv[value]
 
 if res.nil?
   raise "RECORD NOT FOUND"
 else
   puts tsv[value].report
-end
 

data/share/rbbt_commands/tsv/info

@@ -1,4 +1,4 @@
-#!/usr/bin/env ruby
+#"!/usr/bin/env ruby
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
@@ -18,6 +18,7 @@ Display summary information. Works with Tokyocabinet HDB and BDB as well.
 -hh--header_hash* Change the character used to mark the header line (defaults to #)
 -k--key_field* Change the key field
 -f--fields* Change the fields to load
+-s--sep* Change the fields separator (default TAB)
 -h--help Help
 EOF
 
@@ -31,6 +32,7 @@ raise ParameterException, "Please specify the tsv file as argument" if file.nil?
 
 options[:fields] = options[:fields].split(/,\|/) if options[:fields]
 options[:header_hash] = options["header_hash"]
+options[:sep] = options["sep"]
 
 case
 when options[:tokyocabinet]
@@ -53,10 +55,16 @@ else
       puts "  - #{Log.color :cyan, i + 1}: " << Log.color(:yellow, f)
     end
   end
-  puts "Rows: #{Log.color :blue, `wc -l '#{ file }'|cut -f 1 -d' '`}" unless Open.remote? file
-  parts = []
-  header.first_line.split(header.sep).each_with_index{|p,i| parts << (Log.color(:cyan, "(#{i}) ") << p.strip) }
-  puts parts * "\t"
+
+  if String === file and not Open.remote? file and File.exists? file
+    rows = `wc -l '#{ file }' 2>/dev/null|cut -f 1 -d' '`
+  else
+    rows = "Could not get rows of #{Misc.fingerprint file}"
+  end
+    puts "Rows: #{Log.color :blue, rows}" 
+    parts = []
+    header.first_line.split(header.sep).each_with_index{|p,i| parts << (Log.color(:cyan, "(#{i}) ") << p.strip) }
+    puts parts * "\t"
   puts
 end
 

data/share/rbbt_commands/tsv/subset

@@ -15,7 +15,7 @@ Subsets entries from a TSV file from a given list. Works with Tokyocabinet HDB a
 -tch--tokyocabinet File is a TC HDB
 -tcb--tokyocabinet_bd File is a TC BDB
 -hh--header_hash* Change the character used to mark the header line (defaults to #)
--s--subset* Subset of samples (Comma-separated of file)
+-s--subset* Subset of keys (Comma-separated or file)
 -h--help Help
 EOF
 

data/share/rbbt_commands/workflow/info

@@ -14,11 +14,13 @@ Examine the info of a job result
 $ rbbt workflow info <job-result>
 
 -h--help Help
+-a--all Print all info entries
 EOF
 
 SOPT.usage if options[:help]
 
 file = ARGV.shift
+all = options.delete :all
 
 def get_step(file)
   file = file.sub(/\.(info|files)/,'')
@@ -39,6 +41,19 @@ def status_msg(status)
   Log.color(color, status)
 end
 
+def pid_msg(pid, done = false)
+  color = if pid and (done or Misc.pid_exists? pid)
+            :green
+          else
+            :red
+          end
+  if pid.nil?
+    ""
+  else
+    Log.color(color, pid)
+  end
+end
+
 step = get_step file
 
 info = step.info
@@ -47,9 +62,14 @@ inputs = info[:inputs]
 status = info[:status]
 time = info[:time_elapsed]
 messages = info[:messages]
+backtrace = info[:backtrace]
+pid = info[:pid]
+exception = info[:exception]
+rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid]
 
 puts Log.color(:magenta, "File") << ": " << step.path
 puts Log.color(:magenta, "Status") << ": " << status_msg(status)
+puts Log.color(:magenta, "Pid") << ": " << pid_msg(pid, status.to_s == "done")
 puts Log.color(:magenta, "Time") << ": " << time.to_i.to_s << " sec." if time
 if inputs and inputs.any?
   puts Log.color(:magenta, "Inputs")
@@ -81,3 +101,15 @@ if messages and messages.any?
     puts "  " << msg
   end 
 end
+
+if status == :error
+  puts Log.color(:magenta, "Backtrace") << ": "
+  puts Log.color_stack backtrace
+end
+
+if all
+  puts Log.color(:magenta, "Other entries") << ": "
+  rest.each do |key|
+    puts Misc.format_definition_list_item(key, info[key].to_s)
+  end
+end