rbbt-util 3.0.2 → 3.0.3

data/lib/rbbt/util/log.rb

@@ -16,13 +16,15 @@ module Log
     @@severity
   end
 
-  SEVERITY_COLOR = ["0;37m", "32m", "33m", "31m", "1;0m" ].collect{|e| "\033[#{e}"}
+  SEVERITY_COLOR = ["0;37m", "0;32m", "0;33m", "0;31m", "1;0m" ].collect{|e| "\033[#{e}"}
 
   def self.log(message, severity = MEDIUM)
     severity_color = SEVERITY_COLOR[severity]
+    font_color = {false => "\033[0;37m", true => "\033[0m"}[severity >= INFO]
+
     STDERR.puts caller.select{|l| l =~ /rbbt/} * "\n" if @@severity == -1 and not message.empty?
     #STDERR.puts "#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}[#{severity.to_s}]: " +  message if severity >= @@severity
-    STDERR.puts "\033[0;37m#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}#{severity_color}[#{severity.to_s}]\033[0m: " +  message if severity >= @@severity
+    STDERR.puts "\033[0;37m#{Time.now.strftime("[%m/%d/%y-%H:%M:%S]")}#{severity_color}[#{severity.to_s}]\033[0m:#{font_color} " <<  message.strip  << "\033[0m" if severity >= @@severity
   end
 
   def self.debug(message)
@@ -41,6 +43,10 @@ module Log
     log(message, HIGH)
   end
 
+  def self.info(message)
+    log(message, INFO)
+  end
+
   def self.warn(message)
     log(message, WARN)
   end

data/lib/rbbt/util/misc.rb

@@ -1,4 +1,5 @@
 require 'iconv'
+require 'lockfile'
 require 'digest/md5'
 
 class RBBTError < StandardError
@@ -14,9 +15,32 @@ class RBBTError < StandardError
   end
 end
 
+module IndiferentHash
+  def indiferent_get(key)
+    old_get(key) || 
+    old_get(key.to_s) || 
+    old_get(key.to_sym) 
+  end
+
+  def self.extended(base)
+    class << base
+      alias_method :old_get, :[]
+      alias_method :[], :indiferent_get
+    end
+  end
+end
+
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  def self.lock(file, *args)
+    FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
+    lockfile = Lockfile.new file + '.lock'
+    lockfile.lock do
+      yield file, *args
+    end
+  end
+
   def self.string2const(string)
     return nil if string.nil?
     mod = Kernel
@@ -137,15 +161,19 @@ module Misc
 
   def self.hash2md5(hash)
     o = {}
-    hash.each do |k,v|
-      if v.inspect =~ /:0x0/
+    hash.keys.sort_by{|k| k.to_s}.each do |k|
+      v = hash[k]
+      case
+      when v.inspect =~ /:0x0/
         o[k] = v.inspect.sub(/:0x[a-f0-9]+@/,'')
+      when Resource::Path === v
+        "" << String.new(v.to_s)
       else
         o[k] = v
       end
     end
 
-    Digest::MD5.hexdigest(o.inspect)
+    Digest::MD5.hexdigest(o.sort_by{|k| k.to_s}.inspect)
   end
 
   def self.string2hash(string)

data/lib/rbbt/util/open.rb

@@ -200,9 +200,21 @@ module Open
     end
   end
 
-  def self.write(file, content)
+  def self.write(file, content = nil)
     FileUtils.mkdir_p File.dirname(file)
-    if String === content
+    case
+    when content.nil?
+      begin
+        File.open(file, 'w') do |f| 
+          f.flock(File::LOCK_EX)
+          yield f
+          f.flock(File::LOCK_UN)
+        end
+      rescue Exception
+        FileUtils.rm file if File.exists? file
+        raise $!
+      end
+    when String === content
       File.open(file, 'w') do |f|
         f.flock(File::LOCK_EX)
         f.write content 
@@ -217,8 +229,9 @@ module Open
           end
           f.flock(File::LOCK_UN)
         end
-      rescue
+      rescue Exception
         FileUtils.rm file if File.exists? file
+        raise $!
       end
       content.close
     end

data/lib/rbbt/util/persistence.rb

@@ -181,6 +181,7 @@ module Persistence
         FileUtils.rm persistence_file
       end
 
+      Log.debug "Dump data into '#{persistence_file}'"
       per = Persistence::TSV.get persistence_file, true, serializer
 
       per.write
@@ -227,30 +228,41 @@ module Persistence
 
       serializer = tsv_serializer res, extra
 
-      per = Persistence::TSV.get persistence_file, true, serializer
+      begin
+        per = Persistence::TSV.get persistence_file, true, serializer
 
-      per.write
-      per.merge! res
-      Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
-        if extra.include?(key.to_sym)  and per.respond_to?(key.to_sym)
-          per.send "#{key}=".to_sym, extra[key.to_sym]
+        per.write
+        per.merge! res
+        Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+          if extra.include?(key.to_sym)  and per.respond_to?(key.to_sym)
+            per.send "#{key}=".to_sym, extra[key.to_sym]
+          end
         end
+      rescue Exception
+        per.close
+        raise $!
       end
+
       per.read
 
      [ per, extra ]
     else
       Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
-      per = Persistence::TSV.get persistence_file, true, serializer
-
-      extra = {}
-      Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
-        if per.respond_to?(key.to_sym)
-          extra[key] = per.send(key.to_sym)
+      begin
+        per = Persistence::TSV.get persistence_file, true, serializer
+
+        extra = {}
+        Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+          if per.respond_to?(key.to_sym)
+            extra[key] = per.send(key.to_sym)
+          end
         end
-      end
 
-     [ per, extra ]
+      rescue Exception
+        per.close
+        raise $!
+      end
+      [ per, extra ]
     end
   end
 
@@ -314,16 +326,16 @@ module Persistence
 
     filename = get_filename(file)
 
-    if persistence == :no_create
-      o = options.dup
-      options = 
-        Misc.add_defaults options, :persistence_update => false, :persistence_file => nil, :filename => nil
-      persistence_update, persistence_dir, persistence_file, filename =
-        Misc.process_options options, :persistence_update, :persistence_dir, :persistence_file, :filename
+    o = options.dup
+    o = 
+      Misc.add_defaults o, :persistence_update => false, :persistence_file => nil, :filename => nil
+    persistence_update, persistence_dir, persistence_file, filename =
+      Misc.process_options o, :persistence_update, :persistence_dir, :persistence_file, :filename
 
-      filename         ||= get_filename(file)
-      persistence_file ||= get_persistence_file(filename, prefix, options.merge(:persistence_dir => persistence_dir))
+    filename         ||= get_filename(file)
+    persistence_file ||= get_persistence_file(filename, prefix, o.merge(:persistence_dir => persistence_dir))
 
+    if persistence == :no_create
       persistence = false if not File.exists? persistence_file
     end
 
@@ -333,21 +345,24 @@ module Persistence
     else
       Log.low "Persistent Loading for #{filename}. Prefix: #{prefix}. Type #{persistence_type.to_s}"
 
-      case persistence_type.to_sym
-      when :string
-        persist_string(file, prefix, options, &block)
-      when :marshal
-        persist_marshal(file, prefix, options, &block)
-      when :yaml
-        persist_yaml(file, prefix, options, &block)
-      when :tsv
-        persist_tsv(file, prefix, options, &block)
-      when :tsv_string
-        persist_tsv_string(file, prefix, options, &block)
-      when :tsv_extra
-        persist_tsv_extra(file, prefix, options, &block)
-      when :fwt
-        persist_fwt(file, prefix, options, &block)
+      Misc.lock(persistence_file, file, prefix, options, block) do |persistence_file,file,prefix,options,block|
+
+        case persistence_type.to_sym
+        when :string
+          persist_string(file, prefix, options, &block)
+        when :marshal
+          persist_marshal(file, prefix, options, &block)
+        when :yaml
+          persist_yaml(file, prefix, options, &block)
+        when :tsv
+          persist_tsv(file, prefix, options, &block)
+        when :tsv_string
+          persist_tsv_string(file, prefix, options, &block)
+        when :tsv_extra
+          persist_tsv_extra(file, prefix, options, &block)
+        when :fwt
+          persist_fwt(file, prefix, options, &block)
+        end
       end
     end
   end

data/lib/rbbt/util/resource.rb

@@ -240,6 +240,14 @@ source "$INSTALL_HELPER_FILE"
       Open.read self.find, *args
     end
 
+    def yaml(*args)
+      YAML.load(open)
+    end
+
+    def marshal(*args)
+      Marshal.load(open)
+    end
+
     def write(content, *args)
       FileUtils.mkdir_p File.dirname(self.find) unless File.exists? self.find
       Open.write(self.find, content, *args)

data/lib/rbbt/util/task.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/open'
+require 'rbbt/util/task/job'
 
 class Task
   class << self
@@ -7,246 +8,12 @@ class Task
 
   @basedir = "."
 
-  class Job
-    attr_accessor :task, :id, :name, :options, :dependencies, :pid, :path, :previous_jobs
-
-    IDSEP = "_"
-
-    def self.id2name(job_id)
-      job_id.split(IDSEP)
-    end
-    
-    def self.load(task, id)
-      name, hash = id2name(id)
-      self.new task, id, name, nil, nil
-    end
-
-    def initialize(task, id, name, options, dependencies)
-      @task = task  
-      @id =id
-      @name = name
-      @options = options
-      @dependencies = dependencies
-
-      if @dependencies
-        @previous_jobs = Hash[*dependencies.first.collect{|job| job.task.name}.zip(dependencies.first).flatten]
-        dependencies.first.collect{|job| @previous_jobs.merge! job.previous_jobs }
-      else
-        @previous_jobs = []
-      end
-
-      basedir = task.workflow.jobdir unless task.workflow.nil?
-      @path = File.join(basedir || Task.basedir, task.name, id)
-    end
-
-    def info_file
-      path + '.info'
-    end
-
-    def info
-      return {} if not File.exists?(info_file)
-      YAML.load(File.open(info_file))
-    end
-
-    def set_info(key, value)
-      i = self.info
-      new_info = i.merge(key => value)
-      Open.write(info_file, new_info.to_yaml)
-    end
-
-    def step(name = nil, message = nil)
-      if name.nil?
-        info[:step]
-      else
-        set_info(:step, name)
-        set_info(:messages, info[:messages] || [] << message) if not message.nil?
-      end
-    end
-    
-    def messages
-      info[:messages] || []
-    end
-
-    def done?
-      [:done, :error, :aborted].include? info[:step]
-    end
-
-    def error?
-      step == :error or step == :aborted
-    end
-
-    def arguments
-      options.values_at *task.options
-    end
-
-    def block
-      task.block
-    end
-
-    def run_dependencies
-      jobs, files = dependencies
-      jobs.each do |job| job.start unless File.exists? job.path; job.step(:done) end
-      files.each do |file| file.produce end
-    end
-
-    def input
-      dep = dependencies.flatten.first
-      if Job === dep
-        dep.load
-      else
-        dep.read
-      end
-    end
-
-    def start
-      Log.medium("Starting Job '#{ name }'. Path: '#{ path }'")
-      set_info(:start_time, Time.now)
-      save_options(options)
-
-      extend task.scope unless task.scope.nil? or Object == task.scope.class
-
-      begin
-
-        if dependencies.flatten.any?
-          run_dependencies
-        end
-
-        result = instance_exec *arguments, &block
-        
-
-        if not result.nil?
-          case task.persistence
-          when nil, :string, :tsv, :integer
-            Open.write(path, result.to_s)
-          when :marshal
-            Open.write(path, Marshal.dump(result))
-          when :yaml
-            Open.write(path, YAML.dump(result))
-          end
-        end
-
-        set_info(:end_time, Time.now)
-        Log.medium("Finished Job '#{ name }'. Path: '#{ path }'")
-      rescue Exception
-        step(:error, "#{$!.class}: #{$!.message}")
-        raise $!
-      end
-    end
-
-    def save_options(options)
-      new_options = {}
-      options.each do |key, value|
-        case 
-        when TSV === value
-          new_options[key] = value.to_s
-        else
-          new_options[key] = value
-        end
-      end
-      set_info(:options, new_options)
-    end
-
-    def recursive_done?
-      previous_jobs.values.inject(true){|acc,j| acc and j.recursive_done?} and done?
-    end
-
-    def run
-      return self if recursive_done?
-      begin
-        step(:started)
-        start
-        step(:done)
-      rescue Exception
-        Log.debug $!.message
-        Log.debug $!.backtrace * "\n"
-        step(:error, "#{$!.class}: #{$!.message}")
-      end
-      self
-    end
-
-    def fork
-      return self if recursive_done?
-      @pid = Process.fork do
-        begin
-          step(:started)
-          start
-          step(:done)
-        rescue Exception
-          Log.debug $!.message
-          Log.debug $!.backtrace * "\n"
-          step(:error, "#{$!.class}: #{$!.message}")
-        end
-        exit
-      end
-
-      self
-    end
-
-    def join
-      if @pid.nil?
-        while not done? do
-          Log.debug "Waiting: #{info[:step]}"
-          sleep 5
-        end
-      else
-        Process.waitpid @pid
-      end
-
-      self
-    end
-
-    def open
-      File.open(path)
-    end
-
-    def read
-      File.open(path) do |f| f.read end
-    end
-
-    def load
-      case task.persistence
-      when :float
-        Open.read(path).to_f
-      when :integer
-        Open.read(path).to_i
-      when :string
-        Open.read(path)
-      when :tsv
-        TSV.new(path)
-      when :marshal
-        Marshal.load(Open.read(path))
-      when :yaml
-        YAML.load(Open.read(path))
-      end
-    end
-
-    def clean
-      FileUtils.rm path if File.exists? path
-      FileUtils.rm info_file if File.exists? info_file
-    end
-
-    def recursive_clean
-      dependencies.first.each do |job| job.recursive_clean end
-      clean
-    end
-  end # END Job
-
   def load(job_id)
     Job.load(self, job_id)
   end
 
-  def job_options(run_options = nil)
-    return {} if options.nil?
-
-    job_options = {}
-    options.each do |option|
-      job_options[option] = Misc.process_options run_options, option
-    end
-
-    job_options
-  end
-
-  def job_id(name, job_options)
+  def job_id(name, job_options, previous_jobs)
+    job_options = job_options.merge :previous_jobs => previous_jobs.collect{|job| job.id} if previous_jobs.any?
     if job_options.any?
       name.to_s + "_" + Misc.hash2md5(job_options)
     else
@@ -254,8 +21,8 @@ class Task
     end
   end
 
-  attr_accessor :name, :persistence, :options, :option_descriptions, :option_types, :option_defaults, :workflow, :dependencies, :scope, :block
-  def initialize(name, persistence = nil, options = nil, option_descriptions = nil, option_types = nil, option_defaults = nil, workflow = nil, dependencies = nil, scope = nil, &block)
+  attr_accessor :name, :persistence, :options, :option_descriptions, :option_types, :option_defaults, :workflow, :dependencies, :scope, :description, :block
+  def initialize(name, persistence = nil, options = nil, option_descriptions = nil, option_types = nil, option_defaults = nil, workflow = nil, dependencies = nil, scope = nil, description = nil, &block)
     dependencies = [dependencies] unless dependencies.nil? or Array === dependencies
     @name = name.to_s
 
@@ -269,68 +36,61 @@ class Task
     @workflow = workflow
     @dependencies = dependencies || []
     @scope = scope
+    @description = description
 
     @block = block unless not block_given?
   end
 
-  def recursive_options
-    all_options         = []
-    option_descriptions = {}
-    option_types        = {}
-    option_defaults     = {}
-
-		all_options.concat           self.options               if   self.options
-		option_descriptions.merge!   self.option_descriptions   if   self.option_descriptions
-		option_types.merge!          self.option_types          if   self.option_types
-		option_defaults.merge!       self.option_defaults       if   self.option_defaults
+  def process_options(args, optional_args)
+    run_options = {}
 
-    self.dependencies.each do |task|
-      task = case
-             when Symbol === task
-               workflow.tasks[task]
-             when Task === task
-               task
-             else
-               next
-             end
-
-      n_all_options, n_option_descriptions, n_option_types, n_option_defaults = task.recursive_options
-
-			all_options.concat           n_all_options           if   n_all_options
-			option_descriptions.merge!   n_option_descriptions   if   n_option_descriptions
-			option_types.merge!          n_option_types          if   n_option_types
-			option_defaults.merge!       n_option_defaults       if   n_option_defaults
-		end
-
-    [all_options, option_descriptions, option_types, option_defaults]
+    options.each do |option|
+      if option_defaults and option_defaults.include? option
+        run_options[option] = Misc.process_options(optional_args, option) || option_defaults[option]
+      else
+        run_options[option] = args.shift
+      end
+    end unless options.nil?
+ 
+    [run_options, args, optional_args]
   end
 
-  def job_dependencies(jobname, run_options = {})
-    jobs = []
-    files = []
+  def setup(jobname, args, optional_args, dependencies)
+    previous_jobs = []
+    required_files = []
+  
+    run_options, args, optional_args = process_options args, optional_args
+
     dependencies.each do |dependency|
       case
+      when Proc === dependency
+        previous_jobs << dependency.call(jobname, run_options)
       when Task === dependency
-        jobs << dependency.job(jobname, run_options)
+        previous_jobs << dependency.job(jobname, *(args + [optional_args]))
+      when Task::Job === dependency
+        previous_jobs << dependency
       when Symbol === dependency
-        raise "No workflow defined, yet dependencies include Symbols (other tasks)" if workflow.nil?
-        jobs << workflow.tasks[dependency].job(jobname, run_options)
+        previous_jobs << workflow.tasks[dependency].job(jobname, *(args + [optional_args]))
       else
-        files << dependency
+        required_files << dependency
       end
     end
-    [jobs, files]
-  end
 
-  def job(jobname, run_options = {})
+    [previous_jobs, required_files, run_options]
+  end
 
-    job_id = self.job_id jobname, run_options
+  def job(jobname, *args)
+    if Hash === args.last
+      optional_args = args.pop
+    else
+      optional_args = {}
+    end
 
-    job_options = self.job_options run_options
+    previous_jobs, required_files, run_options = setup(jobname, args, optional_args, dependencies)
 
-    dependencies = self.job_dependencies(jobname, run_options) 
+    job_id = self.job_id jobname, run_options, previous_jobs
 
-    Job.new(self, job_id, jobname, job_options, dependencies)
+    Job.new(self, job_id, jobname, run_options, previous_jobs, required_files, previous_jobs.first)
   end
 
   def run(*args)