rbbt-util 3.0.3 → 3.1.0

data/bin/rbbt_Rutil.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'rbbt-util'
+
+STDOUT.write Rbbt.share.lib.R["util.R"].find

data/bin/rbbt_exec.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+code_file = ARGV[0]
+output = ARGV[1]
+
+require 'rbbt-util'
+
+code = case 
+       when (code_file.nil? or code_file == '-')
+         STDIN.read
+       else
+         Open.read(code_file)
+       end
+
+data = eval code
+
+data = data.to_s(:sort, true) if TSV === data
+
+case
+when (output.nil? or output == '-')
+  puts data
+when output == "file"
+  if Misc.filename? data
+    tmpfile = data
+  else
+    tmpfile = TmpFile.tmp_file
+    Open.write(tmpfile, data)
+  end
+
+  puts tmpfile
+else
+  Open.write(output, data)
+end

data/lib/rbbt/util/R.rb

@@ -25,7 +25,7 @@ module R
 end
 
 class TSV
-  def R(script)
+  def R(script, open_options = {})
     TmpFile.with_file do |f|
       Open.write(f, self.to_s)
       Log.debug(R.run(
@@ -35,7 +35,8 @@ data = rbbt.tsv('#{f}');
 rbbt.tsv.write('#{f}', data);
       EOF
       ).read)
-      TSV.new(f, :type => :list)
+      open_options = Misc.add_defaults open_options, :type => :list
+      TSV.new(f, open_options)
     end
   end
 end

data/lib/rbbt/util/cmd.rb

@@ -29,16 +29,28 @@ module CMD
           original_close
         end
 
+        def force_close
+          if @pid
+            Log.debug "Forcing close by killing '#{@pid}'"
+            Process.kill("KILL", @pid)
+            Process.waitpid(@pid)
+          end
+          @post.call if @post
+          original_close
+        end
+ 
         alias original_read read
         def read
           data = Misc.fixutf8(original_read)
           self.close unless self.closed?
           data
         end
+
       }
       io
     end
-  end
+
+ end
 
   def self.process_cmd_options(options = {})
     string = ""
@@ -109,7 +121,6 @@ module CMD
       rescue Exception
         raise CMDError, $!.message
       end
-
     }
     sin.first.close
     sout.last.close

data/lib/rbbt/util/fix_width_table.rb

@@ -190,6 +190,8 @@ class FixWidthTable
 
     idx += 1 unless pos(idx) >= r_start
 
+    return [] if idx >= size
+
     values = []
     l_start = pos(idx)
     l_end   = pos_end(idx)

data/lib/rbbt/util/misc.rb

@@ -33,6 +33,10 @@ end
 module Misc
   class FieldNotFoundError < StandardError;end
 
+  def self.filename?(filename)
+    String === filename and filename.length < 1024 and filename.index("\n").nil? and File.exists? filename
+  end
+
   def self.lock(file, *args)
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
     lockfile = Lockfile.new file + '.lock'
@@ -162,12 +166,13 @@ module Misc
   def self.hash2md5(hash)
     o = {}
     hash.keys.sort_by{|k| k.to_s}.each do |k|
+      next if k == :monitor or k == "monitor" or k == :in_situ_persistence or k == "in_situ_persistence"
       v = hash[k]
       case
       when v.inspect =~ /:0x0/
         o[k] = v.inspect.sub(/:0x[a-f0-9]+@/,'')
       when Resource::Path === v
-        "" << String.new(v.to_s)
+        o[k] = "" << String.new(v.to_s)
       else
         o[k] = v
       end
@@ -230,6 +235,7 @@ module Misc
         File.open(path, 'w') do |f|  end
       end
     rescue Interrupt
+      FileUtils.rm_f path
       raise "Interrupted (Ctrl-c)"
     rescue Exception
       FileUtils.rm_f path
@@ -373,6 +379,7 @@ class NamedArray < Array
   end
 
   def zip_fields
+    return [] if self.empty?
     zipped = self[0].zip(*self[1..-1])
     zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
     zipped 

data/lib/rbbt/util/open.rb

@@ -104,7 +104,7 @@ module Open
         CMD.cmd("grep", "-E" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
       end
     else
-      CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true)
+      CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
     end
   end
   
@@ -122,7 +122,7 @@ module Open
     if String === stream
       Zlib::Inflate.inflate(stream)
     else
-      CMD.cmd("gunzip", :pipe => true, :in => stream)
+      CMD.cmd("gunzip", :pipe => true, :in => stream, :post => proc{stream.force_close if stream.respond_to? :force_close})
     end
   end
 

data/lib/rbbt/util/persistence.rb

@@ -23,7 +23,7 @@ module Persistence
   def self.get_persistence_file(file, prefix, options = {})
     persistence_dir = Misc.process_options options, :persistence_dir
     persistence_dir ||= CACHEDIR
-    name = prefix.to_s << ":" << file.to_s << ":"
+    name = prefix.to_s.dup << ":" << file.to_s << ":"
 
     options_md5 = Misc.hash2md5 options
     File.join(persistence_dir, name.to_s.gsub(/\s/,'_').gsub(/\//,'>') + options_md5)
@@ -193,10 +193,10 @@ module Persistence
         end
       end
 
-      per.read
-
       tsv = Object::TSV.new per
 
+      per.read
+
       tsv
     else
       Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
@@ -208,6 +208,8 @@ module Persistence
           tsv.send "#{key}=".to_sym, per.send(key.to_sym) 
         end
       end
+      
+      per.read
 
       tsv
     end
@@ -228,28 +230,32 @@ module Persistence
 
       serializer = tsv_serializer res, extra
 
-      begin
-        per = Persistence::TSV.get persistence_file, true, serializer
-
-        per.write
-        per.merge! res
-        Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
-          if extra.include?(key.to_sym)  and per.respond_to?(key.to_sym)
-            per.send "#{key}=".to_sym, extra[key.to_sym]
+      per = nil
+      if not Persistence::TSV === res
+        begin
+          per = Persistence::TSV.get persistence_file, true, serializer
+
+          per.write
+          per.merge! res
+          Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
+            if extra.include?(key.to_sym)  and per.respond_to?(key.to_sym)
+              per.send "#{key}=".to_sym, extra[key.to_sym]
+            end
           end
+
+        rescue Exception
+          per.close
+          raise $!
         end
-      rescue Exception
-        per.close
-        raise $!
+      else
+        per = res
       end
 
-      per.read
-
      [ per, extra ]
     else
       Log.debug "Loading #{ persistence_file }. Prefix = #{prefix}"
       begin
-        per = Persistence::TSV.get persistence_file, true, serializer
+        per = Persistence::TSV.get persistence_file, false, serializer
 
         extra = {}
         Persistence::TSV::FIELD_INFO_ENTRIES.keys.each do |key| 
@@ -258,10 +264,13 @@ module Persistence
           end
         end
 
+      rescue Interrupt
+        raise "Interrupted"
       rescue Exception
         per.close
         raise $!
       end
+
       [ per, extra ]
     end
   end
@@ -276,7 +285,7 @@ module Persistence
     persistence_file ||= get_persistence_file(filename, prefix, options)
 
     if persistence_update or not File.exists? persistence_file
-      Log.debug "Creating #{ persistence_file }. Prefix = #{prefix}"
+      Log.debug "Creating FWT #{ persistence_file }. Prefix = #{prefix}"
 
       range = options[:range]
 

data/lib/rbbt/util/resource.rb

@@ -180,13 +180,14 @@ source "$INSTALL_HELPER_FILE"
   end
 
   def data_module(klass)
-    relative_to klass, "share/#{self.to_s.downcase}"
+    relative_to klass, "share/#{self.to_s.downcase}" unless klass == base
     rakefile = klass.share.install[self.to_s].Rakefile
     rakefile.lib_dir = Resource.caller_lib_dir
 
+
     self[''].define_as_rake rakefile
     self.namespace = base.to_s
-    self.lib_dir = caller_lib_dir
+    self.lib_dir = Resource.caller_lib_dir
   end
 
   module Path

data/lib/rbbt/util/task.rb

@@ -64,13 +64,16 @@ class Task
     dependencies.each do |dependency|
       case
       when Proc === dependency
-        previous_jobs << dependency.call(jobname, run_options)
-      when Task === dependency
-        previous_jobs << dependency.job(jobname, *(args + [optional_args]))
+        deps = dependency.call(jobname, run_options)
+        if Array === deps
+          previous_jobs.concat deps
+        else
+          previous_jobs << deps
+        end
       when Task::Job === dependency
         previous_jobs << dependency
-      when Symbol === dependency
-        previous_jobs << workflow.tasks[dependency].job(jobname, *(args + [optional_args]))
+      when Task === dependency
+        previous_jobs << dependency.job(jobname, *(args + [optional_args]))
       else
         required_files << dependency
       end
@@ -97,4 +100,73 @@ class Task
     job(*args).start
   end
 
+  def option_info(option)
+    info = {}
+    info[:name] = option
+    info[:source] = name
+    info[:description] = option_descriptions[option] if option_descriptions and option_descriptions.include? option
+    info[:type] = option_types[option] if option_types and option_types.include? option
+    info[:default] = option_defaults[option] if option_defaults and option_defaults.include? option
+
+    info
+  end
+
+  def option_summary
+    options = []
+    optional_options = []
+
+    if self.options
+      self.options.collect{|option|
+        info = option_info(option)
+        if info[:default].nil?
+          options << info
+        else
+          optional_options << info
+        end
+      }
+    end
+
+    dependencies.select{|dep| Task === dep}.each do |task|
+      more_options, more_optional_options = task.option_summary
+      options.concat more_options
+      optional_options.concat more_optional_options
+    end
+
+    [options, optional_options]
+  end
+
+  def usage
+    usage = ""
+    usage << "Task: #{name}\n"
+    usage << "\nDescription: #{description.chomp}\n" if description
+    options, optional_options = option_summary
+
+    if options.any?
+      usage << "\nMandatory options:\n"
+      usage << "\tTask\tName\tType   \tDescription\n"
+      usage << "\t----\t----\t----   \t-----------\n"
+      options.each do |option|
+        option_line = "\t[#{option[:source]}]\t#{option[:name]}"
+        option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
+        option_line << "\t#{option[:description]}" if option[:description]
+        usage << option_line << "\n"
+      end
+    end
+
+    if optional_options.any?
+      usage << "\nOptional options:"
+      usage << "Mandatory options:\n"
+      usage << "\tTask\tName\tDefault  \tType   \tDescription\n"
+      usage << "\t----\t----\t-------  \t----   \t-----------\n"
+      optional_options.each do |option|
+        option_line = "\t[#{option[:source]}]\t#{option[:name]}\t#{option[:default]}"
+        option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
+        option_line << "\t#{option[:description]}" if option[:description]
+        usage << option_line << "\n"
+      end
+    end
+
+    usage
+  end
+
 end

data/lib/rbbt/util/task/job.rb

@@ -82,8 +82,7 @@ class Task
     end
 
     def set_info(key, value)
-      Misc.lock(info_file, key, value) do |info_file, key, value|
-        i = self.info
+      Misc.lock(info_file, key, value) do |info_file, key, value| i = self.info
         new_info = i.merge(key => value)
         Open.write(info_file, new_info.to_yaml)
       end
@@ -108,6 +107,23 @@ class Task
       info[:messages] || []
     end
 
+    def files(file = nil, data = nil)
+      return Dir.glob(File.join(path + '.files/*')).collect{|f| File.basename(f)} if file.nil?
+
+      filename = Resource::Path.path(File.join(path + '.files', file.to_s))
+      if data.nil?
+        filename
+      else
+        Open.write(filename, data)
+      end
+    end
+
+    def abort
+      if @pid
+        Process.kill("INT", @pid)
+      end
+    end
+
     def done?
       [:done, :error, :aborted].include? info[:step]
     end
@@ -244,7 +260,7 @@ class Task
       File.open(path) do |f| f.read end
     end
 
-    def load
+    def load(*args)
       case task.persistence
       when :float
         Open.read(path).to_f
@@ -253,7 +269,7 @@ class Task
       when :string
         Open.read(path)
       when :tsv
-        TSV.new(path)
+        TSV.new(path, *args)
       when :marshal
         Marshal.load(Open.read(path))
       when :yaml

data/lib/rbbt/util/tc_hash.rb

@@ -93,7 +93,7 @@ class TCHash < TokyoCabinet::HDB
         @serializer = Marshal
       else
         mod = Misc.string2const serializer_str
-       @serializer = mod
+        @serializer = mod
       end
     end
   end
@@ -121,6 +121,7 @@ class TCHash < TokyoCabinet::HDB
     @serializer = serializer
 
     if write || ! File.exists?(@path_to_db)
+      self.setcache(100000) or raise "Error setting cache"
       self.open(true)
     else
       self.open(false)