rbbt-util 3.0.2 → 3.0.3

data/lib/rbbt/util/task/job.rb

@@ -0,0 +1,276 @@
+require 'rbbt/util/misc'
+
+class Task
+  class Job
+    attr_accessor :task, :id, :name, :options, :previsous_jobs, :required_files, :pid, :path, :previous_jobs, :input
+
+    IDSEP = "_"
+
+    def self.id2name(job_id)
+      job_id.split(IDSEP)
+    end
+
+    def self.load(task, id)
+      name, hash = id2name(id)
+      job = self.new task, id, name, nil, nil
+      job.load_dependencies
+      job
+    end
+
+    def initialize(task, id, name, options = nil, previous_jobs = nil, required_files = nil, input = nil)
+      @task = task  
+      @id =id
+      @name = name
+      @options = options || {}
+      @previous_jobs = previous_jobs || []
+      @required_files = required_files || []
+      @input = input
+
+      basedir = task.workflow.jobdir unless task.workflow.nil?
+      @path = File.join(basedir || Task.basedir, task.name, id)
+    end
+
+    def previous_jobs_rec
+      return [] if previous_jobs.nil?
+      previous_jobs + previous_jobs.collect{|job| job.previous_jobs_rec}.flatten
+    end
+
+    def previous_jobs=(previous_jobs)
+      @previous_jobs = previous_jobs
+      @all_inputs = nil
+    end
+
+    def all_inputs
+      if true or not defined? @all_inputs
+        @all_inputs = {}
+        previous_jobs_rec.each do |job| @all_inputs[job.task.name] = job end
+        @all_inputs.extend IndiferentHash
+        @all_inputs
+      else
+        @all_inputs
+      end
+    end
+
+    def input(name = nil)
+      if name.nil?
+        if @input.nil?
+          nil
+        else
+          @input.load
+        end
+      else
+        all_inputs[name]
+      end
+    end 
+
+    def previous_jobs
+      if @previous_jobs.nil?
+        nil
+      else
+        NamedArray.name @previous_jobs, @previous_jobs.collect{|job| job.task.name} 
+      end
+    end
+
+    def info_file
+      path + '.info'
+    end
+
+    def info
+      return {} if not File.exists?(info_file)
+      info = YAML.load(File.open(info_file))
+      info.extend IndiferentHash
+    end
+
+    def set_info(key, value)
+      Misc.lock(info_file, key, value) do |info_file, key, value|
+        i = self.info
+        new_info = i.merge(key => value)
+        Open.write(info_file, new_info.to_yaml)
+      end
+    end
+
+    def step(name = nil, message = nil)
+      @previous_jobs
+      if name.nil?
+        info[:step]
+      else
+        set_info(:step, name)
+        if message.nil?
+          Log.info "[#{task.name}] Step '#{name}'"
+        else
+          Log.info "[#{task.name}] Step '#{name}': #{message.chomp}"
+          set_info(:messages, info[:messages] || [] << message) if not message.nil?
+        end
+      end
+    end
+
+    def messages
+      info[:messages] || []
+    end
+
+    def done?
+      [:done, :error, :aborted].include? info[:step]
+    end
+
+    def error?
+      step == :error or step == :aborted
+    end
+
+    def arguments
+      options.values_at *task.options
+    end
+
+    def block
+      task.block
+    end
+
+    def run_dependencies
+      required_files.each do |file| file.produce unless File.exists? file end unless required_files.nil?
+      previous_jobs.each do |job| job.start unless File.exists? job.path; job.set_info(:step, :done) end unless previous_jobs.nil?
+    end
+
+    def save_dependencies
+      set_info :previous_jobs, @previous_jobs.collect{|job| "JOB:#{job.task.name}/#{job.id}"}  unless @previous_jobs.nil?
+      set_info :required_files, @required_files.collect{|file| file.responds_to? :find ? file.find : file} if @required_files.nil?
+    end
+
+    def load_dependencies
+      @previous_jobs = info[:previous_jobs].collect do |job_string| 
+        job_string =~ /JOB:(.*)\/(.*)/
+        task.workflow.load_job($1, $2)
+      end if info[:previous_jobs]
+      @required_files = info[:required_files] if info[:required_files]
+    end
+
+    def start
+      begin
+        run_dependencies
+
+        Log.medium("[#{task.name}] Starting Job '#{ name }'. Path: '#{ path }'")
+        set_info(:start_time, Time.now)
+        save_options(options)
+        save_dependencies
+
+        extend task.scope unless task.scope.nil? or Object == task.scope.class
+
+        result = instance_exec *arguments, &block
+
+        if not result.nil?
+          case task.persistence
+          when nil, :string, :tsv, :integer
+            Open.write(path, result.to_s)
+          when :marshal
+            Open.write(path, Marshal.dump(result))
+          when :yaml
+            Open.write(path, YAML.dump(result))
+          end
+        end
+
+        set_info(:end_time, Time.now)
+        Log.medium("[#{task.name}] Finished Job '#{ name }'. Path: '#{ path }'")
+      rescue Exception
+        step(:error, "#{$!.class}: #{$!.message}")
+        raise $!
+      end
+    end
+
+    def save_options(options)
+      new_options = {}
+      options.each do |key, value|
+        case 
+        when TSV === value
+          new_options[key] = value.to_s
+        else
+          new_options[key] = value
+        end
+      end
+      set_info(:options, new_options)
+    end
+
+    def recursive_done?
+      previous_jobs.inject(true){|acc,j| acc and j.recursive_done?} and done?
+    end
+
+    def run
+      return self if recursive_done?
+      begin
+        step(:started)
+        start
+        step(:done)
+      rescue Exception
+        Log.debug $!.message
+        Log.debug $!.backtrace * "\n"
+        step(:error, "#{$!.class}: #{$!.message}")
+      end
+      self
+    end
+
+    def fork
+      return self if recursive_done?
+      @pid = Process.fork do
+        begin
+          step(:started)
+          start
+          step(:done)
+        rescue Exception
+          Log.debug $!.message
+          Log.debug $!.backtrace * "\n"
+          step(:error, "#{$!.class}: #{$!.message}")
+        end
+        exit
+      end
+
+      self
+    end
+
+    def join
+      if @pid.nil?
+        while not done? do
+          Log.debug "Waiting: #{info[:step]}"
+          sleep 5
+        end
+      else
+        Process.waitpid @pid
+      end
+
+      self
+    end
+
+    def open
+      File.open(path)
+    end
+
+    def read
+      File.open(path) do |f| f.read end
+    end
+
+    def load
+      case task.persistence
+      when :float
+        Open.read(path).to_f
+      when :integer
+        Open.read(path).to_i
+      when :string
+        Open.read(path)
+      when :tsv
+        TSV.new(path)
+      when :marshal
+        Marshal.load(Open.read(path))
+      when :yaml
+        YAML.load(Open.read(path))
+      end
+    end
+
+    def clean
+      FileUtils.rm path if File.exists? path
+      FileUtils.rm info_file if File.exists? info_file
+    end
+
+    def recursive_clean
+      previous_jobs.each do |job| job.recursive_clean end unless previous_jobs.nil?
+      clean
+    end
+  end # END Job
+end
+
+

data/lib/rbbt/util/tsv/attach.rb

@@ -1,4 +1,46 @@
 class TSV
+  def self.merge_rows(input, output, sep = "\t")
+    is = case
+         when (String === input and not input.index("\n") and input.length < 250 and File.exists?(input))
+           CMD.cmd("sort -k1,1 -t'#{sep}' #{ input } | grep -v '^#{sep}' ", :pipe => true)
+         when (String === input or StringIO === input)
+           CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => input, :pipe => true)
+         else
+           input
+         end
+ 
+    current_key  = nil
+    current_parts = []
+
+    done = false
+    Open.write(output) do |os|
+
+      done = is.eof?
+      while not done
+        key, *parts = is.gets.sub("\n",'').split(sep, -1)
+        current_key ||= key
+        case
+        when key.nil?
+        when current_key == key
+          parts.each_with_index do |part,i|
+            if current_parts[i].nil?
+              current_parts[i] = part
+            else
+              current_parts[i] = current_parts[i] << "|" << part
+            end
+          end
+        when current_key != key
+          os.puts [current_key, current_parts].flatten * sep
+          current_key = key
+          current_parts = parts
+        end
+
+        done = is.eof?
+      end
+
+    end
+  end
+
   def self.paste_merge(file1, file2, output, sep = "\t")
     case
     when (String === file1 and not file1.index("\n") and file1.length < 250 and File.exists?(file1))
@@ -186,7 +228,7 @@ class TSV
       if other.include? key
         new_values = other[key].values_at *fields
         new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
-        new_values.collect!{|v| v.first} if not type == :double and     other.type == :double
+        new_values.collect!{|v| v.nil? ? nil : v.first} if not type == :double and     other.type == :double
         self[key] = self[key].concat new_values
       else
         if type == :double
@@ -223,8 +265,8 @@ class TSV
             end
           end
 
-          new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
-          new_values.collect!{|v| v.first} if not type == :double and     other.type == :double
+          new_values.collect!{|v| [v]}                    if     type == :double and not other.type == :double
+          new_values.collect!{|v| v.nil? ? nil : v.first} if not type == :double and     other.type == :double
           all_new_values << new_values
         end
       end
@@ -274,7 +316,7 @@ class TSV
             end
           end
           new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
-          new_values.collect!{|v| v.first} if not type == :double and     other.type == :double
+          new_values.collect!{|v| v.nil? ? nil : v.first} if not type == :double and     other.type == :double
           all_new_values << new_values
         end
       end
@@ -330,8 +372,8 @@ class TSV
   end
 
   def self.build_traverse_index(files, options = {})
-    options      = Misc.add_defaults options, :in_namespace => false, :persist_input => false
-    in_namespace = options[:in_namespace]
+    options       = Misc.add_defaults options, :in_namespace => false, :persist_input => false
+    in_namespace  = options[:in_namespace]
     persist_input = options[:persist_input]
 
     path = find_path(files, options)
@@ -339,26 +381,42 @@ class TSV
     return nil if path.nil?
     
     traversal_ids = path.collect{|p| p.first}
-    
-    Log.medium "Found Traversal: #{traversal_ids * " => "}"
-
-    current_id, current_file = path.shift
-    current_key = current_file.all_fields.first
 
-    index   = current_file.index :target => current_id, :fields =>  current_key, :persistence => persist_input
+    Log.medium "Found Traversal: #{traversal_ids * " => "}"
+    
+    data_key, data_file = path.shift
+    if data_key == data_file.key_field
+      Log.debug "Data index not required '#{data_file.key_field}' => '#{data_key}'"
+      data_index = nil
+    else
+      Log.debug "Data index required"
+      data_index = data_file.index :target => data_key, :fields => data_file.key_field, :persistence => false
+    end
 
+    current_index = data_index
+    current_key   = data_key
     while not path.empty?
-      current_id, current_file = path.shift
-      current_index   = current_file.index :target => current_id, :fields => index.fields.first, :persistence => true
-      index.process 0 do |value|
-        current_index.values_at(*value).flatten.uniq
+      next_key, next_file = path.shift
+
+      if current_index.nil?
+        current_index = next_file.index :target => next_key, :fields => current_key, :persistence => persist_input
+      else
+        next_index = next_file.index :target => next_key, :fields => current_key, :persistence => persist_input
+        current_index.process current_index.fields.first do |key, values, values|
+          if values.nil?
+            nil
+          else
+            next_index.values_at(*values).flatten.collect
+          end
+        end
+        current_index.fields = [next_key]
       end
-      index.fields = current_index.fields
     end
 
-    index
+    current_index
   end
 
+
   def self.find_traversal(tsv1, tsv2, options = {})
     options      = Misc.add_defaults options, :in_namespace => false
     in_namespace = options[:in_namespace]
@@ -388,14 +446,23 @@ class TSV
     in_namespace = options[:in_namespace]
 
     fields = other.fields - [key_field].concat(self.fields) if fields == :all
-    fields = other.fields_in_namespace - [key_field].concat(self.fields) if fields.nil?
+    if in_namespace
+      fields = other.fields_in_namespace - [key_field].concat(self.fields) if fields.nil?
+    else
+      fields = other.fields - [key_field].concat(self.fields) if fields.nil?
+    end
+
     Log.high("Attaching fields:#{fields.inspect} from #{other.filename.inspect}.")
+
+    other = other.tsv(:persistence => options[:persist_input] == true) unless TSV === other 
     case
     when key_field == other.key_field
       attach_same_key other, fields
     when (not in_namespace and self.fields.include?(other.key_field))
+      Log.medium "Found other's key field: #{other.key_field}"
       attach_source_key other, other.key_field, fields
     when (in_namespace and self.fields_in_namespace.include?(other.key_field))
+      Log.medium "Found other's key field in #{in_namespace}: #{other.key_field}"
       attach_source_key other, other.key_field, fields
     else
       index = TSV.find_traversal(self, other, options)

data/lib/rbbt/util/workflow.rb

@@ -1,6 +1,8 @@
 require 'rbbt/util/resource'
 require 'rbbt/util/task'
 require 'rbbt/util/persistence'
+require 'rbbt/util/misc'
+
 module WorkFlow
   def self.extended(base)
     class << base
@@ -11,10 +13,16 @@ module WorkFlow
     base.extend Resource
     base.lib_dir = Resource.caller_base_dir if base.class == Object
     base.tasks = {}
+    base.tasks.extend IndiferentHash
     base.jobdir = (File.exists?(base.var.find(:lib)) ? base.var.find(:lib) : base.var.find)
     base.clear_dangling
   end
 
+  def tasks=(tasks)
+    tasks.extend IndiferentHash
+    @tasks = tasks
+  end
+
   def local_persist(*args, &block)
     argsv = *args
     options = argsv.pop
@@ -34,6 +42,7 @@ module WorkFlow
     @dangling_option_types = {} 
     @dangling_option_defaults = {}
     @dangling_dependencies = nil
+    @dangling_description = nil
   end
 
   def task_option(*args)
@@ -49,13 +58,18 @@ module WorkFlow
     @dangling_dependencies = dependencies
   end
 
+  def task_description(description)
+    @dangling_description = description
+  end
+
   def process_dangling
     res = [ 
       @dangling_options, 
       Hash[*@dangling_options.zip(@dangling_option_descriptions.values_at(*@dangling_options)).flatten],
       Hash[*@dangling_options.zip(@dangling_option_types.values_at(*@dangling_options)).flatten],
       Hash[*@dangling_options.zip(@dangling_option_defaults.values_at(*@dangling_options)).flatten],
-      @dangling_dependencies || @last_task,
+      (@dangling_dependencies || [@last_task]).compact,
+      @dangling_description,
     ]
 
     clear_dangling
@@ -70,37 +84,17 @@ module WorkFlow
       persistence = :marshal
     end
 
-    options, option_descriptions, option_types, option_defaults, dependencies = process_dangling
+    options, option_descriptions, option_types, option_defaults, dependencies, description = process_dangling
     option_descriptions.delete_if do |k,v| v.nil? end
     option_types.delete_if do |k,v| v.nil? end
     option_defaults.delete_if do |k,v| v.nil? end
-    task = Task.new name, persistence, options, option_descriptions, option_types, option_defaults, self, dependencies, self, &block
+    task = Task.new name, persistence, options, option_descriptions, option_types, option_defaults, self, dependencies, self, description, &block
     tasks[name] = task
     @last_task = task
   end
 
   def job(task, jobname, *args)
-    task = tasks[task]
-    raise "Task #{ task } not found" if task.nil?
-
-    all_options, option_descriptions, option_types, option_defaults = task.recursive_options
-
-    non_optional_arguments = all_options.reject{|option| option_defaults.include? option}
-    run_options = nil
-
-    case
-    when args.length == non_optional_arguments.length
-      run_options = Hash[*non_optional_arguments.zip(args).flatten].merge option_defaults
-    when args.length == non_optional_arguments.length + 1
-      optional_args = args.pop
-      run_options = option_defaults.
-        merge(optional_args).
-        merge(Hash[*non_optional_arguments.zip(args).flatten])
-    else
-      raise "Number of non optional arguments (#{non_optional_arguments * ', '}) does not match given (#{args.flatten * ", "})"
-    end
-
-    task.job(jobname,  run_options)
+    tasks[task].job(jobname, *args)
   end
 
   def run(*args)