rbbt-util 2.1.0 → 3.0.2

data/lib/rbbt/util/simpleDSL.rb

@@ -67,7 +67,7 @@ module SimpleDSL
   def load_config(method = nil, file = nil, &block)
     @config = {}
     if file
-      raise ConfigFileMissingError.new "File '#{ file }' is missing. Have you installed the config files? (use rbbt_config)." unless File.exists? file
+      raise ConfigFileMissingError.new "Config file '#{ file }' is missing" unless File.exists? file
       parse(method, file)
     end
 

data/lib/rbbt/util/simpleopt.rb

@@ -8,7 +8,7 @@ module SOPT
     opts.split(/:/).each do |opt|
       short, long = opt.sub(/\*$/,'').split('--').values_at(0,1)
       i= {
-        :arg => !opt.match(/\*$/).nil?,
+        :arg => !!opt.match(/\*$/),
       }
       i[:short] = short unless short.nil? || short.empty?
       i[:long] = '--' + long unless long.nil? || long.empty?

data/lib/rbbt/util/task.rb

@@ -0,0 +1,340 @@
+require 'rbbt/util/open'
+
+class Task
+  class << self
+    attr_accessor :basedir
+  end
+
+  @basedir = "."
+
+  class Job
+    attr_accessor :task, :id, :name, :options, :dependencies, :pid, :path, :previous_jobs
+
+    IDSEP = "_"
+
+    def self.id2name(job_id)
+      job_id.split(IDSEP)
+    end
+    
+    def self.load(task, id)
+      name, hash = id2name(id)
+      self.new task, id, name, nil, nil
+    end
+
+    def initialize(task, id, name, options, dependencies)
+      @task = task  
+      @id =id
+      @name = name
+      @options = options
+      @dependencies = dependencies
+
+      if @dependencies
+        @previous_jobs = Hash[*dependencies.first.collect{|job| job.task.name}.zip(dependencies.first).flatten]
+        dependencies.first.collect{|job| @previous_jobs.merge! job.previous_jobs }
+      else
+        @previous_jobs = []
+      end
+
+      basedir = task.workflow.jobdir unless task.workflow.nil?
+      @path = File.join(basedir || Task.basedir, task.name, id)
+    end
+
+    def info_file
+      path + '.info'
+    end
+
+    def info
+      return {} if not File.exists?(info_file)
+      YAML.load(File.open(info_file))
+    end
+
+    def set_info(key, value)
+      i = self.info
+      new_info = i.merge(key => value)
+      Open.write(info_file, new_info.to_yaml)
+    end
+
+    def step(name = nil, message = nil)
+      if name.nil?
+        info[:step]
+      else
+        set_info(:step, name)
+        set_info(:messages, info[:messages] || [] << message) if not message.nil?
+      end
+    end
+    
+    def messages
+      info[:messages] || []
+    end
+
+    def done?
+      [:done, :error, :aborted].include? info[:step]
+    end
+
+    def error?
+      step == :error or step == :aborted
+    end
+
+    def arguments
+      options.values_at *task.options
+    end
+
+    def block
+      task.block
+    end
+
+    def run_dependencies
+      jobs, files = dependencies
+      jobs.each do |job| job.start unless File.exists? job.path; job.step(:done) end
+      files.each do |file| file.produce end
+    end
+
+    def input
+      dep = dependencies.flatten.first
+      if Job === dep
+        dep.load
+      else
+        dep.read
+      end
+    end
+
+    def start
+      Log.medium("Starting Job '#{ name }'. Path: '#{ path }'")
+      set_info(:start_time, Time.now)
+      save_options(options)
+
+      extend task.scope unless task.scope.nil? or Object == task.scope.class
+
+      begin
+
+        if dependencies.flatten.any?
+          run_dependencies
+        end
+
+        result = instance_exec *arguments, &block
+        
+
+        if not result.nil?
+          case task.persistence
+          when nil, :string, :tsv, :integer
+            Open.write(path, result.to_s)
+          when :marshal
+            Open.write(path, Marshal.dump(result))
+          when :yaml
+            Open.write(path, YAML.dump(result))
+          end
+        end
+
+        set_info(:end_time, Time.now)
+        Log.medium("Finished Job '#{ name }'. Path: '#{ path }'")
+      rescue Exception
+        step(:error, "#{$!.class}: #{$!.message}")
+        raise $!
+      end
+    end
+
+    def save_options(options)
+      new_options = {}
+      options.each do |key, value|
+        case 
+        when TSV === value
+          new_options[key] = value.to_s
+        else
+          new_options[key] = value
+        end
+      end
+      set_info(:options, new_options)
+    end
+
+    def recursive_done?
+      previous_jobs.values.inject(true){|acc,j| acc and j.recursive_done?} and done?
+    end
+
+    def run
+      return self if recursive_done?
+      begin
+        step(:started)
+        start
+        step(:done)
+      rescue Exception
+        Log.debug $!.message
+        Log.debug $!.backtrace * "\n"
+        step(:error, "#{$!.class}: #{$!.message}")
+      end
+      self
+    end
+
+    def fork
+      return self if recursive_done?
+      @pid = Process.fork do
+        begin
+          step(:started)
+          start
+          step(:done)
+        rescue Exception
+          Log.debug $!.message
+          Log.debug $!.backtrace * "\n"
+          step(:error, "#{$!.class}: #{$!.message}")
+        end
+        exit
+      end
+
+      self
+    end
+
+    def join
+      if @pid.nil?
+        while not done? do
+          Log.debug "Waiting: #{info[:step]}"
+          sleep 5
+        end
+      else
+        Process.waitpid @pid
+      end
+
+      self
+    end
+
+    def open
+      File.open(path)
+    end
+
+    def read
+      File.open(path) do |f| f.read end
+    end
+
+    def load
+      case task.persistence
+      when :float
+        Open.read(path).to_f
+      when :integer
+        Open.read(path).to_i
+      when :string
+        Open.read(path)
+      when :tsv
+        TSV.new(path)
+      when :marshal
+        Marshal.load(Open.read(path))
+      when :yaml
+        YAML.load(Open.read(path))
+      end
+    end
+
+    def clean
+      FileUtils.rm path if File.exists? path
+      FileUtils.rm info_file if File.exists? info_file
+    end
+
+    def recursive_clean
+      dependencies.first.each do |job| job.recursive_clean end
+      clean
+    end
+  end # END Job
+
+  def load(job_id)
+    Job.load(self, job_id)
+  end
+
+  def job_options(run_options = nil)
+    return {} if options.nil?
+
+    job_options = {}
+    options.each do |option|
+      job_options[option] = Misc.process_options run_options, option
+    end
+
+    job_options
+  end
+
+  def job_id(name, job_options)
+    if job_options.any?
+      name.to_s + "_" + Misc.hash2md5(job_options)
+    else
+      name.to_s
+    end
+  end
+
+  attr_accessor :name, :persistence, :options, :option_descriptions, :option_types, :option_defaults, :workflow, :dependencies, :scope, :block
+  def initialize(name, persistence = nil, options = nil, option_descriptions = nil, option_types = nil, option_defaults = nil, workflow = nil, dependencies = nil, scope = nil, &block)
+    dependencies = [dependencies] unless dependencies.nil? or Array === dependencies
+    @name = name.to_s
+
+    @persistence = persistence || :marshal
+
+    @options = Array === options ? options : [options] unless options.nil? 
+
+    @option_defaults = option_defaults 
+    @option_descriptions = option_descriptions 
+    @option_types = option_types 
+    @workflow = workflow
+    @dependencies = dependencies || []
+    @scope = scope
+
+    @block = block unless not block_given?
+  end
+
+  def recursive_options
+    all_options         = []
+    option_descriptions = {}
+    option_types        = {}
+    option_defaults     = {}
+
+		all_options.concat           self.options               if   self.options
+		option_descriptions.merge!   self.option_descriptions   if   self.option_descriptions
+		option_types.merge!          self.option_types          if   self.option_types
+		option_defaults.merge!       self.option_defaults       if   self.option_defaults
+
+    self.dependencies.each do |task|
+      task = case
+             when Symbol === task
+               workflow.tasks[task]
+             when Task === task
+               task
+             else
+               next
+             end
+
+      n_all_options, n_option_descriptions, n_option_types, n_option_defaults = task.recursive_options
+
+			all_options.concat           n_all_options           if   n_all_options
+			option_descriptions.merge!   n_option_descriptions   if   n_option_descriptions
+			option_types.merge!          n_option_types          if   n_option_types
+			option_defaults.merge!       n_option_defaults       if   n_option_defaults
+		end
+
+    [all_options, option_descriptions, option_types, option_defaults]
+  end
+
+  def job_dependencies(jobname, run_options = {})
+    jobs = []
+    files = []
+    dependencies.each do |dependency|
+      case
+      when Task === dependency
+        jobs << dependency.job(jobname, run_options)
+      when Symbol === dependency
+        raise "No workflow defined, yet dependencies include Symbols (other tasks)" if workflow.nil?
+        jobs << workflow.tasks[dependency].job(jobname, run_options)
+      else
+        files << dependency
+      end
+    end
+    [jobs, files]
+  end
+
+  def job(jobname, run_options = {})
+
+    job_id = self.job_id jobname, run_options
+
+    job_options = self.job_options run_options
+
+    dependencies = self.job_dependencies(jobname, run_options) 
+
+    Job.new(self, job_id, jobname, job_options, dependencies)
+  end
+
+  def run(*args)
+    job(*args).start
+  end
+
+end

data/lib/rbbt/util/tc_hash.rb

@@ -5,6 +5,17 @@ class TCHash < TokyoCabinet::HDB
   class OpenError < StandardError;end
   class KeyFormatError < StandardError;end
 
+  class IntegerSerializer
+    def self.dump(i); [i].pack("l"); end
+    def self.load(str); str.unpack("l").first; end
+  end
+
+  class IntegerArraySerializer
+    def self.dump(a); a.pack("l*"); end
+    def self.load(str); str.unpack("l*"); end
+  end
+
+
   class StringSerializer
     def self.dump(str); str.to_s; end
     def self.load(str); str; end
@@ -31,7 +42,7 @@ class TCHash < TokyoCabinet::HDB
   end
 
 
-  ALIAS = {:marshal => Marshal, nil => Marshal, :single => StringSerializer, :list => StringArraySerializer, :double => StringDoubleArraySerializer}
+  ALIAS = {:integer => IntegerSerializer, :integer_array => IntegerArraySerializer, :marshal => Marshal, nil => Marshal, :single => StringSerializer, :list => StringArraySerializer, :double => StringDoubleArraySerializer}
 
   CONNECTIONS = {}
 
@@ -179,7 +190,13 @@ class TCHash < TokyoCabinet::HDB
   
   def collect
     res = []
-    self.each{|k, v| res << yield(k,v)}
+    self.each{|k, v| 
+      if block_given?
+        res << yield(k,v)
+      else
+        res << [k,v]
+      end
+    }
     res
   end
 

data/lib/rbbt/util/tsv.rb

@@ -1,6 +1,6 @@
+require 'rbbt/util/resource'
 require 'rbbt/util/misc'
 require 'rbbt/util/open'
-require 'rbbt/util/path'
 require 'rbbt/util/tc_hash'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/log'
@@ -13,6 +13,7 @@ require 'rbbt/util/tsv/accessor'
 require 'rbbt/util/tsv/manipulate'
 require 'rbbt/util/tsv/index'
 require 'rbbt/util/tsv/attach'
+require 'rbbt/util/tsv/resource'
 class TSV
 
   def self.headers(file, options = {})
@@ -65,7 +66,7 @@ class TSV
 
     @filename = Misc.process_options options, :filename
     @filename ||= case
-                  when Path === file
+                  when Resource::Path === file
                     file
                   when (String === file and File.exists? file)
                     File.expand_path file
@@ -89,13 +90,18 @@ class TSV
     #         TSV: Duplicate
     case
     when block_given?
-      @data, extra = Persistence.persist(@filename, :TSV, :tsv_extra, options.merge(:force_array => true)) do |filename, options| yield filename, options end
+      @data, extra = Persistence.persist(file, :TSV, :tsv_extra, options.merge(:force_array => true)) do |file, options, filename| yield file, options, filename end
       extra.each do |key, values|
         self.send("#{ key }=".to_sym, values) if self.respond_to? "#{ key }=".to_sym 
       end if not extra.nil?
  
     else
+
       case
+      when Array === file
+        @data = Hash[file.collect{|v| 
+          [v,[]]
+        }]
       when Hash === file 
         @data = file
       when TSV === file
@@ -108,12 +114,12 @@ class TSV
           end
         end
       else
-        @data, extra = Persistence.persist(@filename, :TSV, :tsv_extra, options) do |file, options, filename|
+        @data, extra = Persistence.persist(file, :TSV, :tsv_extra, options) do |file, options, filename|
           data, extra = nil
 
           case
             ## Parse source
-          when (String === file and file.respond_to? :open)
+          when Resource::Path === file #(String === file and file.respond_to? :open)
             data, extra = TSV.parse(file.open(:grep => options[:grep]) , options)
             extra[:namespace] ||= file.namespace
             extra[:datadir]   ||= file.datadir
@@ -123,14 +129,13 @@ class TSV
             Open.open(file, :grep => options[:grep]) do |f|
               data, extra = TSV.parse(f, options)
             end
-            #extra[:namespace] = File.basename(File.dirname(filename))
-            #extra.delete :namespace if extra[:namespace].empty? or extra[:namespace] == "."
           when File === file
+            path = file.path
+            file = Open.grep(file, options[:grep]) if options[:grep]
+            data, extra = TSV.parse(file, options)
+          when IO === file
             file = Open.grep(file, options[:grep]) if options[:grep]
             data, extra = TSV.parse(file, options)
-            extra[:namespace] = File.basename(File.dirname(file.filename))
-            extra.delete :namespace if extra[:namespace].empty? or extra[:namespace] == "."
-            ## Encapsulate Hash or TSV
           when block_given?
             data 
           else