rbbt 1.1.7 → 2.0.0

data/lib/rbbt/util/filecache.rb

@@ -1,72 +0,0 @@
-require 'fileutils'
-require 'rbbt'
-
-# Provides caching functionality for files downloaded from the internet
-module FileCache
-
-  class BadPathError    < StandardError; end
-  class FileExistsError < StandardError; end
-
-  private 
-
-  # Remove slash characters from filename.
-  def self.clean_path(filename)
-    filename.gsub(/\//,'_SLASH_')
-  end
-
-  # Check that the file name is safe and is in the correct format
-  def self.sanity_check(filename)
-    if filename =~ /\//
-      raise FileCache::BadPathError, "Character / not allowed in name: #{ filename }"
-    end
-    if filename !~ /.+\..+/
-      raise FileCache::BadPathError, "Filename '#{filename}' must have name and extension: name.ext"
-    end
-  end
-
-  public 
-
-  # Find the path that a particular file would have in the cache
-  def self.path(filename)
-    sanity_check(filename)
-
-    name, extension = filename.match(/(.+)\.(.+)/).values_at(1,2)
-    dirs = name.scan(/./).reverse.values_at(0,1,2,3,4).reverse.compact.join('/')
-                                            
-    return File.join(File.join(Rbbt.cachedir,dirs),filename)
-  end
-
-  # Add a file in the cache. Raise exception if exists, unless force is
-  # used.
-  def self.add_file(filename, content, options = {})
-    sanity_check(filename)
-
-    path = path(filename)
-    FileUtils.makedirs(File.dirname(path), :mode => 0777)
-
-    if File.exist?(path) and ! (options[:force] || options['force'])
-      raise FileCache::FileExistsError, "File #{filename} already in cache"
-    end
-
-    File.open(path,'w'){|f|
-      f.write(content)
-    }
-    FileUtils.chmod 0666, path
-
-    nil
-  end
-
-  # Removes the file from cache
-  def self.del_file(filename)
-    sanity_check(filename)
-
-    path = path(filename)
-
-    if File.exist? path
-      FileUtils.rm path
-    end
-
-    nil
-  end
-
-end

data/lib/rbbt/util/index.rb

@@ -1,47 +0,0 @@
-require 'rbbt/util/open'
-require 'rbbt/util/arrayHash'
-
-module Index
-
-  # Creates an inverse index. Takes a file with rows of elements
-  # separated by a given pattern (specified by +sep+) and returns a hash
-  # where each element points to the first element in the row. +lexicon+
-  # is the file containing the data.
-  def self.index(lexicon, options = {}) 
-    options = {:sep => "\t|\\|", :case_sensitive => true}.merge(options)
-
-
-    data = Open.to_hash(lexicon, options)
-    if options[:clean]
-      data = ArrayHash.clean(data)
-    end
-
-    index = {}
-
-    data.each{|code, id_lists|
-      next if code.nil? || code == ""
-      id_lists.flatten.compact.uniq.each{|id|
-        id = id.downcase unless options[:case_sensitive]
-        index[id] = code
-      }
-    }
-    data.each{|code, id_lists|
-      next if code.nil? || code == ""
-      id = code
-      id = id.downcase unless options[:case_sensitive]
-      index[id] = code
-    }
-
-    if !options[:case_sensitive]
-      class << index; self; end.instance_eval{
-        alias_method :old_get, :[]
-        define_method(:[], proc{|key| old_get(key.to_s.downcase)})
-
-        alias_method :old_values_at, :values_at
-        define_method(:values_at, proc{|*keys| old_values_at(*keys.collect{|key| key.to_s.downcase }) })
-      }
-    end
-
-    index
-  end
-end

data/lib/rbbt/util/misc.rb

@@ -1,106 +0,0 @@
-require 'rbbt'
-require 'rbbt/util/open'
-
-class String
-  CONSONANTS = []
-  if File.exists? File.join(Rbbt.datadir, 'wordlists/consonants')
-    Object::Open.read(File.join(Rbbt.datadir, 'wordlists/consonants')).each_line{|l| CONSONANTS << l.chomp}
-  end
-
-  # Uses heuristics to checks if a string seems like a special word, like a gene name.
-  def is_special?
-    # Only consonants
-    return true if self =~ /^[bcdfghjklmnpqrstvwxz]+$/i 
-
-    # Not a word
-    return false if self =~ /[^\s]\s[^\s]/;
-    return false if self.length < 3;
-    # Alphanumeric
-    return true if self =~ /[0-9]/ &&  self =~ /[a-z]/i
-    # All Caps
-    return true if self =~ /[A-Z]{2,}/;
-    # Caps Mix
-    return true if self =~ /[a-z][A-Z]/;
-    # All consonants
-    return true if self =~ /^[a-z]$/i && self !~ /[aeiou]/i
-    # Dashed word
-    return true if self =~ /(^\w-|-\w$)/
-    # To many consonants (very heuristic)
-    if self =~ /([^aeiouy]{3,})/i && !CONSONANTS.include?($1.downcase)
-      return true
-    end
-
-    return false
-  end
-
-  # Turns the first letter to lowercase
-  def downcase_first
-    return "" if self == ""
-    letters = self.scan(/./)
-    letters[0].downcase!
-    letters.join("")
-  end
-
-  # Turns a roman number into arabic form is possible. Just simple
-  # romans only...
-  def arabic
-    return 1 if self =~ /^I$/;
-    return 2 if self =~ /^II$/;
-    return 3 if self =~ /^III$/;
-    return 4 if self =~ /^IV$/;
-    return 5 if self =~ /^V$/;
-    return 10 if self =~ /^X$/;
-
-    return nil
-  end
-end
-
-
-
-
-$greek = {
-    "alpha"   => "a",
-    "beta"    => "b",
-    "gamma"   => "g",
-    "delta"   => "d",
-    "epsilon" => "e",
-    "zeta"    => "z",
-    "eta"     => "e",
-    "theta"   => "th",
-    "iota"    => "i",
-    "kappa"   => "k",
-    "lambda"  => "l",
-    "mu"      => "m",
-    "nu"      => "n",
-    "xi"      => "x",
-    "omicron" => "o",
-    "pi"      => "p",
-    "rho"     => "r",
-    "sigma"   => "s",
-    "tau"     => "t",
-    "upsilon" => "u",
-    "phi"     => "ph",
-    "chi"     => "ch",
-    "psi"     => "ps",
-    "omega"   => "o"
-}
-
-$inverse_greek = Hash.new
-$greek.each{|l,s| $inverse_greek[s] = l }
-
-$stopwords = Open.read(File.join(Rbbt.datadir, 'wordlists/stopwords')).scan(/\w+/) if File.exists? File.join(Rbbt.datadir, 'wordlists/stopwords')
-
-class Array
-
-  # Divides the array into +num+ chunks of the same size by placing one
-  # element in each chunk iteratively.
-  def chunk(num)
-    chunks = []
-    each_with_index{|e, i|
-      c = i % num
-      chunks[c] ||=[]
-      chunks[c] << e
-    }
-    chunks
-  end
-end

data/lib/rbbt/util/open.rb

@@ -1,235 +0,0 @@
-require 'rbbt'
-require 'rbbt/util/tmpfile'
-
-
-# Provides with a few helper functions to read and write files, as well # as
-# for accessing remote files. It supports caching the files.
-module Open
-
-  def self.fields(line, sep = "\t")
-    chunks = line.chomp.split(/(#{sep})/).select{|c| c !~ /^#{sep}$/ }
-    if line =~ /#{sep}$/
-      chunks << ""
-    end
-    chunks
-  end
-
-  class DirectoryNotFoundError < StandardError; end
-  class OpenURLError < StandardError; end
-
-  private 
-
-  @@remote_cachedir =  File.join(Rbbt.cachedir, 'open-remote/')
-  FileUtils.mkdir @@remote_cachedir unless File.exist? @@remote_cachedir
-
-  # If no data is specified and the url is found in the cache the saved
-  # contents are returned, if not found, the url is opened and the contents of
-  # that are returned. If +data+ is specified then it is saved in the
-  # cache under +url+. To match +url+ in the cache a MD5 digest is used.
-  # The location of the cache directory is bu default
-  # File.join(Rbbt.cachedir, 'open-remote/').
-  def self.cache(url, data = nil)
-    require 'digest/md5'
-    digest = Digest::MD5.hexdigest(url)
-
-    if data
-      Open.write(File.join(@@remote_cachedir, digest), data)
-      return nil
-    else
-      if File.exist? File.join(@@remote_cachedir, digest)
-        return File.open(File.join(@@remote_cachedir, digest)){|file| file.read }
-      else
-        return nil
-      end
-    end
-  end
-
-  # Checks if +url+ is a remote file.
-  def self.remote(url)
-    url =~ /^(?:http|ssh|https|ftp):\/\//
-  end
-
-
-  # Checks if +url+ is a gzip file.
-  def self.gziped(url)
-    if remote(url)
-      return url =~ /\.gz$/ || url =~ /\.gz\?.*$/
-    else
-      return url =~ /\.gz$/ 
-    end
-  end
-
-
-  @@last_time = Time.now   
-  def self.wait(lag = 0)
-    time = Time.now   
-
-    if time < @@last_time + lag
-      sleep @@last_time + lag - time
-    end
-
-    @@last_time = Time.now   
-  end
-
-  public
-  # Reads the file specified by url. If the url es local it just opens
-  # the file, if it is remote if checks the cache first. In any case, it
-  # unzips gzip files automatically.
-  #
-  # Options: 
-  # * :quiet    => Do not print the progress of downloads
-  # * :nocache  => do not use the cache.
-  # * :nice     => secconds to wait between online queries
-  #
-  def self.read(url, options = {})
-
-    case
-    when remote(url)
-      if !options[:nocache] && data = cache(url)
-        return data
-      end
-
-      wait(options[:nice]) if options[:nice]
-      tmp = TmpFile.tmp_file("open-")
-      `wget -O #{tmp} '#{url}' #{options[:quiet] ? '-q' : '' }`
-
-      if $?.success?
-        if gziped(url)
-        `mv #{tmp} #{tmp}.gz; gunzip #{tmp}`
-        end
-
-        cache(url, File.open(tmp){|file| file.read}) unless options[:nocache]
-
-        data = File.open(tmp){|file| file.read}
-        FileUtils.rm tmp
-        return data
-      else
-        raise OpenURLError, "Error reading remote url: #{ url }"
-      end
-    
-    when IO === url
-      url.read
-    else
-      return  File.open(url){|file| file.read}
-    end
-
-  end
-
-  # Writes the contents on the path specified by filename
-  #
-  # Options: 
-  # * :force => Create directories if missing.
-  def self.write(filename, content, options = {})
-    if !File.exist? File.dirname(filename) 
-      if options[:force] 
-        FileUtils.makedirs(File.dirname(filename))
-      else
-        raise Open::DirectoryNotFoundError, "Directory #{File.dirname(filename)} was not found"
-      end
-    end
-
-    File.open(filename,'w'){|f|
-      f.write content
-    }
-
-    nil
-  end
-
-  # Writes the contents on the path specified by filename. If the file
-  # is present it appends the contents.
-  #
-  # Options: 
-  # * :force => Create directories if missing.
-  def self.append(filename, content, options ={})
-    if !File.exist? File.dirname(filename) 
-      if options[:force] 
-        FileUtils.makedirs(File.dirname(filename))
-      else
-        raise Open::DirectoryNotFoundError, "Directory #{File.dirname(filename)} was not found"
-      end
-    end
-
-    f = File.open(filename,'a')
-    f.write content
-    f.close
-
-    nil
-  end
-
-
-
-  # Reads a file with rows with elementes separated by a given pattern
-  # and builds a hash with it. The keys of the hash are the elements in
-  # the :native positions, by default the first (0). The value for each
-  # key is an array with one position for each of the rest possible
-  # positions specified in :extra, by default all but the :native. Since
-  # the native key may be repeated, each of the positions of the values
-  # is in itself an array. There are a number of options to change this
-  # behaviour.
-  #
-  # Options:
-  # * :native => position of the elements that will constitute the keys. By default 0.
-  # * :extra => positions of the rest of elements. By default all but :native. It can be an array of positions or a single position.
-  # * :sep =>  pattern to use in splitting the lines into elements, by default "\t"
-  # * :flatten => flatten the array of arrays that hold the values for each key into a simple array.
-  # * :single => for each key select only the first of the values, instead of the complete array.
-  # * :fix  => A Proc that is called to pre-process the line
-  # * :exclude => A Proc that is called to check if the line must be excluded from the process.
-  def self.to_hash(filename, options = {})
-    native  = options[:native]  || 0
-    extra   = options[:extra]
-    exclude = options[:exclude]
-    fix     = options[:fix]
-    sep     = options[:sep]     || "\t"
-    single  = options[:single]  
-    single  = false if single.nil?
-    flatten = options[:flatten] || single
-    flatten = single if flatten.nil?
-
-    extra = [extra] if extra && ! extra.is_a?( Array)
-
-    data = {}
-    Open.read(filename).each_line{|l|
-      l = fix.call(l) if fix
-      next if exclude and exclude.call(l)
-
-      row_fields = self.fields(l, sep)
-      id = row_fields[native]
-      next if id.nil? || id == ""
-
-      data[id] ||= []
-      if extra
-        fields = extra
-      else
-        fields = (0..(row_fields.length - 1)).to_a - [native] 
-      end
-      fields.each_with_index{|pos,i|
-        data[id][i] ||= []
-        data[id][i] << row_fields[pos]
-      }
-    }
-
-    if flatten
-      data.each{|key, values| 
-        if values 
-          values.flatten!
-          values.collect!{|v| 
-            if v != ""
-              v 
-            else 
-              nil 
-            end
-          }
-          values.compact! 
-        else 
-          nil 
-        end
-      } 
-    end
-
-    data = Hash[*(data.collect{|key,values| [key, values.first]}).flatten] if single
-    
-    data
-  end
-
-end