rbbt-util 1.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010-2011 Miguel Vázquez García
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/tchash.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+options = SOPT.parse "-h--help"
+
+command = ARGV.shift
+file    = ARGV.shift
+
+case command
+when 'cat'
+  puts TSV.new(TCHash.get(file))
+end
+

data/bin/tsv.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+
+options = SOPT.parse "-h--help:-to--tsv-options*:-p--persistence"
+
+command = ARGV.shift
+file    = ARGV.shift
+
+case command
+when 'cat'
+  puts TSV.new(file, options["tsv-options"].merge(options["persistence"]))
+when '
+

data/lib/rbbt/util/cachehelper.rb

@@ -0,0 +1,100 @@
+require 'digest'
+
+module CacheHelper
+  CACHE_DIR = '/tmp/cachehelper'
+  FileUtils.mkdir_p(CACHE_DIR) unless File.exist?(CACHE_DIR)
+
+  LOG_TIME = false
+  class CacheLocked < Exception; end
+
+  def self.time(id)
+    t = Time.now
+    data = block.call
+    STDERR.puts "#{ id } time: #{Time.now - t}"
+    data
+  end
+
+  def self.cachedir=(dir)
+    @@cachedir=dir
+    FileUtils.mkdir_p(dir) unless File.exist?(dir)
+  end
+
+  def self.cachedir
+    @@cachedir ||= CACHE_DIR
+  end
+
+
+  def self.reset
+    FileUtils.rm Dir.glob(cachedir + '*')
+  end
+ 
+  def self.reset_locks
+    FileUtils.rm Dir.glob(cachedir + '*.lock')
+  end
+ 
+ 
+  def self.build_filename(name, key)
+    File.join(cachedir, name + ": " + Digest::MD5.hexdigest(key.to_s))
+  end
+ 
+  def self.do(filename, block)
+    FileUtils.touch(filename + '.lock')
+
+    if LOG_TIME
+      data = time do
+        block.call
+      end
+    else
+      data = block.call
+    end
+    
+    File.open(filename, 'w'){|f| f.write data}
+    FileUtils.rm(filename + '.lock')
+    return data
+  end
+ 
+  def self.clean(name)
+    FileUtils.rm Dir.glob(File.join(cachedir, "#{ name }*"))
+  end
+ 
+  def self.cache_ready?(name, key)
+    filename = CacheHelper.build_filename(name, key)
+    File.exist?(filename)
+  end
+ 
+  def self.cache(name, key = [], wait = nil, &block)
+    filename = CacheHelper.build_filename(name, key)
+    begin
+      case
+      when File.exist?(filename)
+        return File.open(filename){|f| f.read}
+      when File.exist?(filename + '.lock')
+        raise CacheLocked
+      else
+        if wait.nil?
+          CacheHelper.do(filename, block)
+        else
+          Thread.new{CacheHelper.do(filename, block)}
+          return wait
+        end
+ 
+      end
+    rescue CacheLocked
+      if wait.nil?
+        sleep 30
+        retry
+      else
+        return wait
+      end
+    rescue Exception
+      FileUtils.rm(filename + '.lock') if File.exist?(filename + '.lock')
+      raise $!
+    end
+  end
+ 
+  def self.marshal_cache(name, key = [])
+    Marshal::load( cache(name, key) do
+      Marshal::dump(yield)
+    end)
+  end
+end

data/lib/rbbt/util/cmd.rb

@@ -0,0 +1,140 @@
+require 'rbbt/util/misc'
+require 'rbbt/util/log'
+require 'stringio'
+
+module CMD
+  class CMDError < StandardError;end
+
+  module SmartIO 
+    def self.tie(io, pid = nil, post = nil)
+      io.instance_eval{
+        @pid  = pid
+        @post = post
+        alias original_close close
+        def close
+          begin
+            Process.waitpid(@pid, Process::WNOHANG) if @pid
+          rescue
+          end
+
+          @post.call if @post
+          original_close
+        end
+
+        alias original_read read
+        def read
+          data = Misc.fixutf8(original_read)
+          self.close unless self.closed?
+          data
+        end
+      }
+      io
+    end
+  end
+
+  def self.process_cmd_options(options = {})
+    string = ""
+    options.each do |option, value|
+      case 
+      when value.nil? || FalseClass === value 
+        next
+      when TrueClass === value
+        string << "#{option} "
+      else
+        if option.chars.to_a.last == "="
+          string << "#{option}#{value} "
+        else
+          string << "#{option} #{value} "
+        end
+      end
+    end
+
+    string.strip
+  end
+
+  def self.cmd(cmd, options = {}, &block)
+    options = Misc.add_defaults options, :stderr => Log::DEBUG
+    in_content = options.delete(:in)
+    stderr     = options.delete(:stderr)
+    pipe       = options.delete(:pipe)
+    post       = options.delete(:post)
+
+    if stderr == true
+      stderr = Log::HIGH
+    end
+
+    # Process cmd_options
+    cmd_options = process_cmd_options options
+    if cmd =~ /'\{opt\}'/
+      cmd.sub!('\'{opt}\'', cmd_options) 
+    else
+      cmd << " " << cmd_options
+    end
+
+    sout, serr = IO.pipe, IO.pipe
+
+    case 
+    when (IO === in_content and not StringIO === in_content)
+      sin = [in_content, nil]
+    else StringIO === in_content
+      sin = IO.pipe
+    end
+
+    pid = fork {
+      begin
+
+        sin.last.close if sin.last
+        STDIN.reopen sin.first
+        sin.first.close
+
+
+        serr.first.close
+        STDERR.reopen serr.last
+        serr.last.close
+
+        sout.first.close
+        STDOUT.reopen sout.last
+        sout.last.close
+
+        STDOUT.sync = STDERR.sync = true
+        exec(cmd)
+      rescue Exception
+        raise CMDError, $!.message
+      end
+
+    }
+    sin.first.close
+    sout.last.close
+    serr.last.close
+
+    case 
+    when String === in_content
+      sin.last.write in_content
+      sin.last.close
+    when StringIO === in_content
+      Thread.new do
+        while l = in_content.gets
+          sin.last.write l
+        end
+        sin.last.close
+      end
+    end
+
+    Thread.new do
+      while l = serr.first.gets
+        Log.log l, stderr if Integer === stderr
+      end
+      serr.first.close
+    end
+
+    if pipe
+      SmartIO.tie sout.first, pid, post
+      sout.first
+    else
+      out = StringIO.new sout.first.read
+      SmartIO.tie out
+      Process.waitpid pid
+      out
+    end
+  end
+end

data/lib/rbbt/util/data_module.rb

@@ -0,0 +1,81 @@
+module DataModule
+
+  def self.extended(base)
+    if defined? base::PKG and base::PKG
+      base.pkg_module = base::PKG
+    else
+      base.pkg_module = Rbbt
+    end
+
+    base.sharedir = PKGData.get_caller_sharedir 
+  end
+
+  def pkg_module
+    @pkg_module
+  end
+
+  def pkg_module=(pkg_module)
+    @pkg_module = pkg_module
+  end
+
+  def sharedir
+    @sharedir
+  end
+
+  def sharedir=(sharedir)
+    @sharedir = sharedir
+  end
+
+  alias old_method_missing method_missing
+  def method_missing(name, *args, &block)
+    if args.any?
+      filename = File.join(self.to_s, args.first, name.to_s)
+    else
+      filename = File.join(self.to_s, name.to_s)
+    end
+
+    begin
+      pkg_module.add_datafiles filename => ['', self.to_s, sharedir]
+    rescue 
+      Log.debug $!.message
+      old_method_missing name, *args, &block
+    end
+
+    pkg_module.find_datafile filename
+  end
+
+  module WithKey
+    def klass=(klass)
+      @klass = klass
+    end
+   
+    def klass
+      @klass
+    end
+
+    def key=(key)
+      @key = key
+    end
+
+    def key
+      @key
+    end
+
+    def method_missing(name, *args)
+      if key
+        klass.send(name, key, *args)
+      else
+        klass.send(name, *args)
+      end
+    end
+  end
+
+  def with_key(key)
+    klass = self
+    o = Object.new
+    o.extend WithKey
+    o.klass = self
+    o.key = key
+    o
+  end
+end

data/lib/rbbt/util/excel2tsv.rb

@@ -0,0 +1,32 @@
+require 'rbbt/util/tsv'
+require 'spreadsheet'
+
+class TSV
+  def self.excel2tsv(file, options = {})
+    sheet = options.delete :sheet
+    header = options.delete :header
+    header = true unless header == false
+    sheet ||= 0
+    TmpFile.with_file do |filename|
+      workbook = Spreadsheet.open File.open(file)
+      sheet    = workbook.worksheet sheet
+
+      rows = []
+
+      sheet.each do |row|
+        rows << row.values_at(0..(row.size - 1))
+      end
+
+      File.open(filename, 'w') do |f|
+        if header
+          header = rows.shift
+          f.puts "#" + header * "\t"
+        end
+
+        rows.each do |row| f.puts row * "\t" end
+      end
+
+      TSV.new(filename, options)
+    end
+  end
+end

data/lib/rbbt/util/filecache.rb

@@ -0,0 +1,58 @@
+require 'fileutils'
+require 'rbbt/util/misc'
+
+# Provides caching functionality for files downloaded from the internet
+module FileCache
+  CACHEDIR = "/tmp/rbbt_cache" 
+  FileUtils.mkdir CACHEDIR unless File.exist? CACHEDIR
+
+  def self.cachedir=(cachedir)
+    CACHEDIR.replace cachedir
+    FileUtils.mkdir_p CACHEDIR unless File.exist? CACHEDIR
+  end
+
+  def self.cachedir
+    CACHEDIR
+  end
+
+  def self.path(filename)
+    filename = File.basename filename
+
+    filename.match(/(.+)\.(.+)/)
+
+    base = filename.sub(/\..+/,'')
+    dirs = base.scan(/./).values_at(0,1,2,3,4).compact.reverse
+
+    File.join(File.join(CACHEDIR, *dirs), filename) 
+  end
+
+  def self.add(filename, content)
+    path = path(filename)
+    
+    FileUtils.makedirs(File.dirname(path), :mode => 0777)
+
+    Misc.sensiblewrite(path, content)
+
+    FileUtils.chmod 0666, path
+
+    path
+  end
+
+  def self.found(filename)
+    File.exists? FileCache.path(filename)
+  end
+
+  def self.get(filename)
+    path = path(filename)
+
+    return nil if ! File.exists? path
+
+    File.open(path)
+  end
+
+  def self.del(filename)
+    path = path(filename)
+
+    FileUtils.rm path if File.exist? path
+  end
+end

data/lib/rbbt/util/log.rb

@@ -0,0 +1,50 @@
+module Log
+
+  DEBUG  = 0
+  LOW    = 1
+  MEDIUM = 2
+  HIGH   = 3
+
+  def severity=(severity)
+    @@severity = severity
+  end
+
+  def severity
+    @@severity
+  end
+
+  def self.log(message, severity = MEDIUM)
+    STDERR.puts "#{Time.now}[#{severity.to_s}]: " +  message if severity >= @@severity
+  end
+
+  def self.debug(message)
+    log(message, DEBUG)
+  end
+
+  def self.low(message)
+    log(message, LOW)
+  end
+
+  def self.medium(message)
+    log(message, MEDIUM)
+  end
+
+  def self.high(message)
+    log(message, HIGH)
+  end
+
+  case ENV['RBBT_LOG']
+  when 'DEBUG' 
+    @@severity = DEBUG
+  when 'LOW' 
+    @@severity = LOW
+  when 'MEDIUM' 
+    @@severity = MEDIUM
+  when 'HIGH' 
+    @@severity = HIGH
+  when nil
+    @@severity = HIGH
+  else
+    @@severity = ENV['RBBT_LOG'].to_i
+  end
+end

data/lib/rbbt/util/misc.rb

@@ -0,0 +1,158 @@
+require 'iconv'
+module Misc
+  class FieldNotFoundError < StandardError;end
+
+  def self.env_add(var, value, sep = ":", prepend = true)
+    ENV[var] ||= ""
+    return if ENV[var] =~ /(#{sep}|^)#{Regexp.quote value}(#{sep}|$)/
+    if prepend
+      ENV[var] = value + sep + ENV[var]
+    else
+      ENV[var] += sep + ENV[var]
+    end
+  end
+
+  def self.count(list)
+    counts = Hash.new 0
+    list.each do |item|
+      counts[item] += 1
+    end
+
+    counts
+  end
+
+  def self.profile
+    require 'ruby-prof'
+    RubyProf.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = RubyProf.stop
+      printer = RubyProf::FlatPrinter.new(result)
+      printer.print(STDOUT, 0)
+    end
+
+    res
+  end
+
+  def self.fixutf8(string)
+    if string.respond_to?(:valid_encoding?) and ! string.valid_encoding?
+      @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')
+      @@ic.iconv(string)
+    else
+      string
+    end
+  end
+
+  def self.add_defaults(options, defaults = {})
+    case
+    when Hash === options
+      new_options = options.dup
+    when String === options
+      new_options = string2hash options
+    else
+      raise "Format of '#{options.inspect}' not understood"
+    end
+    defaults.each do |key, value|
+      new_options[key] = value if new_options[key].nil?
+    end
+    new_options
+  end
+
+  def self.string2hash(string)
+
+    options = {}
+    string.split(/#/).each do |str|
+      if str.match(/(.*)=(.*)/)
+        option, value = $1, $2
+      else
+        option, value = str, true
+      end
+
+      option = option.sub(":",'').to_sym if option.chars.first == ':'
+
+      
+      if value == true
+        options[option] = option.to_s.chars.first != '!' 
+      else
+        options[option] = begin eval(value) rescue value end
+      end
+    end
+
+    options
+  end
+
+  def self.sensiblewrite(path, content)
+    begin
+      case
+      when String === content
+        File.open(path, 'w') do |f|  f.write content  end
+      when (IO === content or StringIO === content)
+        File.open(path, 'w') do |f|  while l = content.gets; f.write l; end  end
+      else
+        File.open(path, 'w') do |f|  end
+      end
+    rescue Interrupt
+      raise "Interrupted (Ctrl-c)"
+    rescue Exception
+      FileUtils.rm_f path
+      raise $!
+    end
+  end
+
+  def self.field_position(fields, field, quiet = false)
+    return field if Integer === field or Range === field
+    raise FieldNotFoundError, "Field information missing" if fields.nil? && ! quiet
+    fields.each_with_index{|f,i| return i if f == field}
+    field_re = Regexp.new /#{field}/i
+    fields.each_with_index{|f,i| return i if f =~ field_re}
+    raise FieldNotFoundError, "Field '#{ field }' was not found" unless quiet
+  end
+end
+
+class NamedArray < Array
+  attr_accessor :fields
+
+  def self.name(array, fields)
+    a = self.new(array)
+    a.fields = fields
+    a
+  end
+
+  def positions(fields)
+    fields.collect{|field|
+      Misc.field_position(@fields, field)
+    }
+  end
+
+  alias original_get_brackets []
+  def [](key)
+    original_get_brackets(Misc.field_position(fields, key))
+  end
+
+  alias original_set_brackets []=
+  def []=(key,value)
+    original_set_brackets(Misc.field_position(fields, key), value)
+  end
+
+  alias original_values_at values_at
+  def values_at(*keys)
+    keys = keys.collect{|k| Misc.field_position(fields, k) }
+    original_values_at(*keys)
+  end
+end
+
+
+def profile 
+  require 'ruby-prof'
+  RubyProf.start
+  yield
+  result = RubyProf.stop
+
+  # Print a flat profile to text
+  printer = RubyProf::FlatPrinter.new(result)
+  printer.print(STDOUT, 0)
+end