rbbt-util 1.0.1

data/lib/rbbt/util/open.rb

@@ -0,0 +1,200 @@
+require 'rbbt/util/cmd'
+require 'rbbt/util/misc'
+require 'rbbt/util/tmpfile'
+
+require 'zlib'
+require 'digest/md5'
+
+module Open
+  class OpenURLError < StandardError; end
+  class OpenGzipError < StandardError; end
+
+  REMOTE_CACHEDIR = "/tmp/open_cache" 
+  FileUtils.mkdir REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
+
+  def self.cachedir=(cachedir)
+    REMOTE_CACHEDIR.replace cachedir
+    FileUtils.mkdir REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
+  end
+
+  def self.cachedir
+    REMOTE_CACHEDIR
+  end
+
+  # Remote WGET
+  LAST_TIME = {}
+  def self.wait(lag, key = nil)
+    time = Time.now   
+
+    if LAST_TIME[key] != nil && (time < LAST_TIME[key] + lag)
+      sleep (LAST_TIME[key] + lag) - time
+    end
+
+    LAST_TIME[key] = Time.now   
+  end
+
+  def self.wget(url, options = {})
+    options = Misc.add_defaults options, "--user-agent=" => 'firefox', :pipe => true
+
+    wait(options[:nice], options[:nice_key]) if options[:nice]
+    options.delete(:nice)
+    options.delete(:nice_key)
+
+    pipe  = options.delete(:pipe)
+    quiet = options.delete(:quiet)
+    options["--quiet"] = quiet if options["--quiet"].nil?
+
+    stderr = case
+             when options['stderr']
+               options['stderr'] 
+             when options['--quiet']
+               false
+             else
+               nil
+             end
+    begin
+      CMD.cmd("wget '#{ url }'", options.merge(
+        '-O' => '-', 
+        :pipe => pipe, 
+        :stderr => stderr
+      ))
+    rescue
+     STDERR.puts $!.backtrace.inspect
+     raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
+    end
+  end
+
+  # Cache
+
+  def self.in_cache(url, options = {})
+    digest = Digest::MD5.hexdigest(url)
+
+    filename = File.join(REMOTE_CACHEDIR, digest)
+    if File.exists? filename
+      return filename 
+    else
+      nil
+    end
+  end
+  
+  def self.add_cache(url, data, options = {})
+    digest = Digest::MD5.hexdigest(url)
+    Misc.sensiblewrite(File.join(REMOTE_CACHEDIR, digest), data)
+  end
+
+  # Grep
+  
+  def self.grep(stream, grep)
+    case 
+    when Array === grep
+      TmpFile.with_file(grep * "\n", false) do |f|
+        CMD.cmd("grep", "-F" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
+      end
+    else
+      CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true)
+    end
+  end
+  
+  def self.file_open(file, grep)
+    if grep
+      grep(File.open(file), grep)
+    else
+      File.open(file)
+    end
+  end
+
+  # Decompression
+   
+  def self.gunzip(stream)
+    if String === stream
+      Zlib::Inflate.inflate(stream)
+    else
+      CMD.cmd("gunzip", :pipe => true, :in => stream)
+    end
+  end
+
+  def self.unzip(stream)
+    TmpFile.with_file(stream.read) do |filename|
+      StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", "-p" => true, :pipe => true).read)
+    end
+  end
+
+  # Questions
+
+  def self.remote?(file)
+    !! (file =~ /^(?:https?|ftp):\/\//)
+  end
+
+  def self.gzip?(file)
+    !! (file =~ /\.gz$/)
+  end
+
+  def self.zip?(file)
+    !! (file =~ /\.zip/)
+  end
+
+  # Open Read Write
+
+  def self.open(url, options = {})
+    options = Misc.add_defaults options, :noz => false
+
+    wget_options = options[:wget_options] || {}
+    wget_options[:nice] = options.delete(:nice)
+    wget_options[:nice_key] = options.delete(:nice_key)
+
+    io = case
+         when (not remote?(url))
+           file_open(url, options[:grep])
+         when options[:nocache]
+           wget(url, wget_options)
+         when in_cache(url)
+           file_open(in_cache(url), options[:grep])
+         else
+           io = wget(url, wget_options)
+           add_cache(url, io)
+           io.close
+           file_open(in_cache(url), options[:grep])
+         end
+    io = unzip(io)  if zip?  url and not options[:noz]
+    io = gunzip(io) if gzip? url and not options[:noz]
+
+    io
+  end
+
+  def self.read(file, options = {}, &block)
+    f = open(file, options)
+
+    if block_given?
+      while l = Misc.fixutf8(f.gets)
+        l = fixutf8(l) if l.respond_to?(:valid_encoding?) && ! l.valid_encoding?
+        yield l
+      end
+    else
+      Misc.fixutf8(f.read)
+    end
+  end
+
+  def self.write(file, content)
+    if String === content
+      File.open(file, 'w') do |f| f.write content end
+    else
+      File.open(file, 'w') do |f| 
+        while l = content.gets
+          f.write l
+        end
+      end
+      content.close
+    end
+  end
+end
+
+if __FILE__ == $0
+  require 'benchmark'
+  require 'progress-monitor'
+
+  file = '/home/mvazquezg/rbbt/data/dbs/entrez/gene_info'
+  puts Benchmark.measure {
+    #Open.open(file).read.split(/\n/).each do |l| l end
+    Open.read(file) do |l| l end
+  }
+end

data/lib/rbbt/util/pkg_config.rb

@@ -0,0 +1,78 @@
+require 'yaml'
+
+module PKGConfig
+  class NoConfig < Exception; end
+  class NoVariables < Exception; end
+
+  class RootdirNotFoundError < StandardError; end
+
+  def self.rootdir_for_file(file = __FILE__)
+    dir = File.expand_path(File.dirname file)
+
+    while not File.exists?(File.join(dir, 'lib')) and dir != '/'
+      dir = File.dirname(dir)
+    end
+
+    if File.exists? File.join(dir, 'lib')
+      dir
+    else
+      raise RootdirNotFoundError
+    end
+  end
+
+  def self.get_caller_rootdir
+    caller.each do |line|
+      next if line =~ /\/pkg_config\.rb/ 
+        begin
+          return PKGConfig.rootdir_for_file(line)
+        rescue RootdirNotFoundError
+        end
+    end
+    raise RootdirNotFoundError
+  end
+
+
+  def self.extended(base)
+    base.module_eval{ @@rootdir = PKGConfig.get_caller_rootdir}
+  end
+
+  def rootdir
+    @@rootdir
+  end
+
+  def load_config(file, pkg_variables)
+    config = YAML.load_file(file)
+
+    pkg_variables.each do |variable|
+      self.send("#{variable}=", config[variable])
+    end
+  end
+
+  def load_cfg(pkg_variables, default = nil)
+    pkg_cfg_files = [ ENV['RBBT_CONFIG'] || "", 
+      File.join(ENV["HOME"], '.' + self.to_s), 
+      File.join('/etc/', '.' +  self.to_s)]
+
+    pkg_variables.each do |variable|
+      self.class_eval %{
+        def self.#{variable} 
+          @#{variable}
+        end
+        def self.#{variable}=(value)
+          @#{variable} = value
+        end
+      }
+    end
+
+    file = pkg_cfg_files.select{|file| File.exists? file}.first
+    if file.nil?
+      if default
+        file = pkg_cfg_files[1]
+        Open.write(file, default)
+      else
+        raise NoConfig, "No config file found. [#{pkg_cfg_files * ", "}]" if file.nil?
+      end
+    end
+    load_config file, pkg_variables
+  end
+end

data/lib/rbbt/util/pkg_data.rb

@@ -0,0 +1,110 @@
+require 'rbbt/util/open'
+require 'rbbt/util/tsv'
+require 'rbbt/util/log'
+require 'rake'
+
+module PKGData
+  FILES = {} unless defined? FILES
+  class SharedirNotFoundError < StandardError; end
+
+  def self.sharedir_for_file(file = __FILE__)
+    dir = File.expand_path(File.dirname file)
+
+    while not File.exists?(File.join(dir, 'lib')) and dir != '/'
+      dir = File.dirname(dir)
+    end
+
+    if File.exists? File.join(dir, 'lib')
+      File.join(dir, 'share')
+    else
+      raise SharedirNotFoundError
+    end
+  end
+
+  def self.get_caller_sharedir
+    caller.each do |line|
+      next if line =~ /\/data_module\.rb/  or line =~ /\/pkg_data\.rb/ 
+        begin
+          return PKGData.sharedir_for_file(line)
+        rescue SharedirNotFoundError
+        end
+    end
+    raise SharedirNotFoundError
+  end
+
+  def run_rake(path, dir, task = nil)
+    rakefile = File.join(dir, 'Rakefile')
+    return nil unless File.exists? rakefile
+    if task.nil?
+      task ||= :default
+    else
+      task.sub!(/\/$/,'') if String === task
+      path = File.dirname(path)
+    end
+
+    load rakefile
+    old_dir = FileUtils.pwd
+    begin
+      FileUtils.mkdir_p path
+      FileUtils.chdir path
+      Rake::Task[task].invoke
+      Rake::Task[task].reenable
+    ensure
+      FileUtils.chdir old_dir
+    end
+    true
+  end
+
+  def get_datafile(file, path, get, sharedir)
+    Log.log "Getting data file '#{ file }' into '#{ path }'. Get: #{get.to_s}"
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    case
+    when get.nil?
+      load File.join(sharedir, 'install', file)
+
+    when Proc === get
+      Open.write(path, get.call(file, path))
+
+    when TSV === get
+      Open.write(path, get.to_s)
+
+    when String === get
+      install_dir =File.expand_path(File.join(sharedir, 'install')) 
+      rake_dir  = File.join(install_dir, File.dirname(get), file)
+      rake_task = nil
+
+      until rake_dir == install_dir
+        return if run_rake(path, rake_dir, rake_task)
+        rake_task = File.join(File.basename(rake_dir), rake_task || "")
+        rake_dir  = File.dirname(rake_dir)
+      end
+
+      if (File.exists?(File.join(sharedir, get)) and not File.directory?(File.join(sharedir, get)))
+        Open.write(path, Open.open(File.join(sharedir, get)))
+      else
+        Open.write(path, Open.open(get, :wget_options => {:pipe => true}, :nocache => true))
+      end
+    end
+  end
+
+  def add_datafiles(files = {})
+    files.each do |file, info|
+      subpath, get, sharedir = info
+
+      path = File.join(datadir, subpath.to_s, file.to_s)
+
+      if not File.exists?(path)
+        sharedir ||= PKGData.get_caller_sharedir
+        get_datafile(file.to_s, path, get, sharedir)
+      end
+
+      FILES[file.to_s] = path
+    end
+  end
+
+  def find_datafile(file)
+    FILES[file.to_s]
+  end
+end

data/lib/rbbt/util/pkg_software.rb

@@ -0,0 +1,130 @@
+require 'rbbt/util/open'
+require 'rbbt/util/tsv'
+require 'rbbt/util/log'
+require 'rbbt/util/cmd'
+require 'rake'
+
+module PKGSoftware
+  SOFTWARE = {} unless defined? SOFTWARE
+
+  class SharedirNotFoundError < StandardError; end
+
+  def self.sharedir_for_file(file = __FILE__)
+    dir = File.expand_path(File.dirname file)
+
+    while not File.exists?(File.join(dir, 'lib')) and dir != '/'
+      dir = File.dirname(dir)
+    end
+
+    if File.exists? File.join(dir, 'lib')
+      File.join(dir, 'share')
+    else
+      raise SharedirNotFoundError
+    end
+  end
+
+  def self.get_caller_sharedir
+    caller.each do |line|
+      next if line =~ /\/data_module\.rb/  or line =~ /\/pkg_software\.rb/ 
+        begin
+          return PKGData.sharedir_for_file(line)
+        rescue SharedirNotFoundError
+        end
+    end
+    raise SharedirNotFoundError
+  end
+
+  def software_dir
+    File.join(datadir, 'software')
+  end
+
+  def opt_dir
+    File.join(software_dir, 'opt')
+  end
+
+  def bin_dir
+    File.join(opt_dir, 'bin')
+  end
+
+
+  def get_pkg(pkg, path, get, sharedir)
+    Log.log "Getting software '#{ pkg }' into '#{ path }'. Get: #{get.to_s}"
+
+    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
+
+    if get.nil? or get.empty?
+      CMD.cmd("#{File.join(sharedir, 'install', 'software', pkg)} #{File.join(Rbbt.rootdir, 'share/install/software/lib', 'install_helpers')} #{software_dir}", :stderr => Log::HIGH)
+    else 
+      CMD.cmd("#{File.join(sharedir, 'install', 'software', get)} #{File.join(Rbbt.rootdir, 'share/install/software/lib', 'install_helpers')} #{software_dir}")
+    end
+  end
+
+  def add_software(pkgs = {})
+    pkgs.each do |pkg, info|
+      subpath, get, sharedir = info
+
+      setup_env(software_dir)
+
+      path = File.join(opt_dir, subpath.to_s, pkg.to_s)
+
+      if not File.exists?(path)
+        sharedir ||= PKGSoftware.get_caller_sharedir
+        get_pkg(pkg.to_s, path, get, sharedir)
+      end
+
+      SOFTWARE[pkg.to_s] = path
+    end
+  end
+
+  def find_software(pkg)
+    SOFTWARE[pkg.to_s]
+  end
+
+
+  def setup_env(software_dir)
+    Misc.env_add 'PATH', bin_dir
+
+    FileUtils.mkdir_p opt_dir unless File.exists? opt_dir
+    %w(.ld-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
+      filename = File.join(opt_dir, file)
+      FileUtils.touch filename unless File.exists? filename
+    end
+
+    if not File.exists?  File.join(opt_dir,'.post_install')
+      Open.write(File.join(opt_dir,'.post_install'),"#!/bin/bash\n")
+    end
+
+    Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
+      Misc.env_add('LD_LIBRARY_PATH',line.chomp)
+      Misc.env_add('LD_RUN_PATH',line.chomp)
+    end
+
+    Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
+      Misc.env_add('PKG_CONFIG_PATH',line.chomp)
+    end
+
+    Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
+      Misc.env_add('LD_LIBRARY_PATH',line.chomp)
+    end
+
+    Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
+      Misc.env_add('LD_LIBRARY_PATH',line.chomp)
+    end
+
+    Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
+      Misc.env_add('ACLOCAL_FLAGS', "-I#{File.join(opt_dir, line.chomp)}", ' ')
+    end
+
+    Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
+      Misc.env_add('CLASSPATH', "#{File.join(opt_dir,'java', 'lib', line.chomp)}")
+    end
+
+    Dir.glob(File.join opt_dir, 'jars', '*').each do |file|
+      Misc.env_add('CLASSPATH', "#{File.expand_path(file)}")
+    end
+
+    File.chmod 0774, File.join(opt_dir, '.post_install')
+
+    CMD.cmd(File.join(opt_dir, '.post_install'))
+  end
+end

data/lib/rbbt/util/simpleDSL.rb

@@ -0,0 +1,92 @@
+# This class helps designing DSL in ruby based on method_missing. Class
+# is initialize with a block of code or a file with the code, and it is
+# given a method to be invoked instead of method missing. This class
+# deals simply with making the method_missing alias and removing it and
+# executing the block of file with code.
+module SimpleDSL
+
+  class ConfigFileMissingError < StandardError; end
+  class NoRuby2Ruby < StandardError; end
+
+  private
+
+  def hook_method(method = nil)
+    method ||= :DSL_action 
+    @@restore_name = ("restore_DSL_" + method.to_s).to_sym
+    @@method_name = method.to_sym
+    
+    class << self
+      @restore_stack ||= []
+      @restore_stack << @@restore_name
+      alias_method(@@restore_name, :method_missing)
+      alias_method(:method_missing, @@method_name)
+    end
+  end
+
+  def unhook_method
+    class << self
+      alias_method(:method_missing, @restore_stack.pop)
+    end
+  end
+
+  public
+
+  def parse(method = nil, actions = nil, &block)
+
+    actions ||= block
+    
+    hook_method(method)
+
+    # Execute
+    if actions.is_a? Proc
+      begin
+        require 'parse_tree_extensions'
+        require 'parse_tree'
+        require 'ruby2ruby'
+        @config[@@method_name] = actions.to_ruby.collect[1..-2].join
+      rescue Exception
+        @config[@@method_name] = NoRuby2Ruby.new "The gem ruby2ruby is not installed. It will not work on ruby 1.9."
+      end
+      
+      instance_eval &actions
+    elsif File.exists?(actions)
+      
+      @config[@@method_name] = File.open(actions).read
+
+      eval File.open(actions).read
+    end
+
+    unhook_method
+
+  end
+
+
+  # Processes a DSL. +method+ is the name of the method executed instead
+  # of method_missing. The code to be evaluated as a DSL is either
+  # specified in +&block+ or in the file pointed by +file+.
+  def load_config(method = nil, file = nil, &block)
+    @config = {}
+    if file
+      raise ConfigFileMissingError.new "File '#{ file }' is missing. Have you installed the config files? (use rbbt_config)." unless File.exists? file
+      parse(method, file)
+    end
+
+    if block
+      parse(method, block)
+    end
+  end
+
+  def config(action = nil)
+    if action
+      config = @config[action.to_sym]
+    else
+      config = @config[:DSL_action]
+    end
+
+    raise config if NoRuby2Ruby === config
+
+    config
+  end
+end
+
+

data/lib/rbbt/util/simpleopt.rb

@@ -0,0 +1,56 @@
+module SOPT
+  def self.name(info)
+    (info[:long] || info[:short]).sub(/^-*/,'')
+  end
+
+  def self.parse(opts)
+    info = {}
+    opts.split(/:/).each do |opt|
+      short, long = opt.sub(/\*$/,'').split('--').values_at(0,1)
+      i= {
+        :arg => !opt.match(/\*$/).nil?,
+      }
+      i[:short] = short unless short.nil? || short.empty?
+      i[:long] = '--' + long unless long.nil? || long.empty?
+      info[name(i)] = i
+    end
+
+    info
+  end
+
+  def self.get(opts)
+    info = parse(opts)
+
+    switches = {}
+    info.each do |name, i|
+      switches[i[:short]] = name if i[:short]
+      switches[i[:long]] = name if i[:long]
+    end
+
+    options = Hash.new(false)
+    rest = []
+
+    index = 0
+    while index < ARGV.length do
+      arg = ARGV[index]
+      if switches.include? arg
+        name = switches[arg]
+        i = info[name]
+        if i[:arg]
+          options[name.to_sym] = ARGV[index + 1]
+          index += 1
+        else
+          options[name.to_sym] = true
+        end
+      else
+        rest << arg
+      end
+      index += 1
+    end
+
+    ARGV.delete_if do true end
+    rest.each do |e| ARGV << e end
+
+    options
+  end
+end