treat 1.0.6 → 1.1.0

data/lib/treat/downloader.rb

@@ -1,87 +0,0 @@
-# Download a file without storing it entirely in memory.
-class Treat::Downloader
-
-  require 'net/http'
-  require 'fileutils'
-
-  class << self
-    attr_accessor :show_progress
-  end
-
-  self.show_progress = false
-
-  MaxTries = 3
-  
-  # Download a file into destination, and return
-  # the path to the downloaded file. If the filename 
-  # is nil, it will set the default filename to 'top'.
-  def self.download(protocol, server, dir, file = nil, target_base = nil, target_dir = nil)
-
-    require 'progressbar' if self.show_progress
-    
-    target_base ||= Treat.files
-    target_dir ||= server
-   
-    dir += '/' if dir && dir[-1] != '/'
-    resource = "#{dir}#{file}"
-    resource = "/#{resource}" unless resource[0] == '/'
-    url = "#{server}#{resource}"
-    path = File.join(target_base, target_dir)
-    
-    unless FileTest.directory?(path)
-      FileUtils.mkdir(path)
-    end
-    
-    
-    file = File.open("#{path}/#{file}", 'w')
-    tries = 0
-    begin
-
-      Net::HTTP.start(server) do |http|
-
-        http.use_ssl = true if protocol == 'https'
-        
-        http.request_get(resource) do |response|
-
-          if response.content_length
-            length = response.content_length
-          else
-            warn 'Unknown file size; ETR unknown.'
-            length = 10000
-          end
-
-          pbar = self.show_progress ?
-          ProgressBar.new(url, length)  : nil
-
-          unless response.code == '200'
-            raise Treat::Exception,
-            "response code was not 200 "+
-            "OK, but was #{response.code}. "
-          end
-
-          response.read_body do |segment|
-            pbar.inc(segment.length) if pbar
-            file.write(segment)
-          end
-
-          pbar.finish if pbar
-
-        end
-
-      end
-
-      file.path.to_s
-
-    rescue Exception => error
-      tries += 1
-      retry if tries < MaxTries
-      raise Treat::Exception,
-      "Couldn't download #{url}. (#{error.message})"
-      file.delete
-    ensure
-      file.close
-    end
-
-  end
-
-end

data/lib/treat/entities/abilities.rb

@@ -1,10 +0,0 @@
-# Requires the -able mixins for the Entity class.
-module Treat::Entities::Abilities
-  
-  p = 'treat/entities/abilities/*.rb'
-  
-  Dir[Treat.lib + p].each do |f| 
-    require f
-  end
-  
-end

data/lib/treat/entities/entities.rb

@@ -1,102 +0,0 @@
-module Treat::Entities
-
-  # Require the generic entity lass.
-  require 'treat/entities/entity'
-
-  # Represents a collection of texts.
-  class Collection < Entity
-    
-    # Initialize the collection with a folder
-    # containing the texts of the collection.
-    def initialize(folder = nil, id = nil)
-      super('', id)
-      set :folder, folder
-      i = folder + '/.index'
-      set :index, i if FileTest.directory?(i)
-    end
-
-    # Works like the default <<, but if the
-    # file being added is a collection or a
-    # document, then copy that collection or
-    # document into this collection's folder.
-    def <<(entities, copy = true)
-      unless entities.is_a? Array
-        entities = [entities]
-      end
-      entities.each do |entity|
-        if [:document, :collection].
-          include?(entity.type) && copy
-          entity = entity.copy_into(self)
-        end
-      end
-      super(entities)
-    end
-
-  end
-
-  # Represents a document.
-  class Document < Entity
-    
-    def initialize(file = nil, id = nil)
-      super('', id)
-      set :file, file
-    end
-
-  end
-
-  # Represents a section, usually with a title
-  # and at least one paragraph.
-  class Section < Entity; end
-
-  # Represents a zone of text
-  # (Title, Paragraph, List, Quote).
-  class Zone < Entity; end
-
-  # Represents a title, subtitle, logical header.
-  class Title < Zone; end
-
-  # Represents a paragraph.
-  class Paragraph < Zone; end
-
-  # Represents a list.
-  class List < Zone; end
-
-  # Represents a group of words.
-  class Phrase < Entity; end
-
-  # Represents a group of words with a sentence ender.
-  class Sentence < Phrase; end
-
-  # Represents a terminal element in the text structure.
-  class Token < Entity; end
-
-  # Represents a word.
-  class Word < Token; end
-
-  # Represents a clitic ('s).
-  class Enclitic < Token; end
-
-  # Represents a number.
-  class Number < Token
-    def to_i; to_s.to_i; end
-    def to_f; to_s.to_f; end
-  end
-
-  # Represents a punctuation sign.
-  class Punctuation < Token; end
-
-  # Represents a character that is neither
-  # alphabetical, numerical or a punctuation
-  # character (e.g. @#$%&*).
-  class Symbol < Token; end
-
-  # Represents a url.
-  class Url < Token; end
-
-  # Represents a valid RFC822 address.
-  class Email < Token; end
-
-  # Represents an entity of unknown type.
-  class Unknown; end
-
-end

data/lib/treat/exception.rb

@@ -1,7 +0,0 @@
-module Treat
-  # Custom exception class for the Treat toolkit.
-  # Used to distinguish between errors raised by
-  # gems/Ruby from errors raised by the toolkit.
-  class Exception < ::Exception; end
-  class InvalidInputException < Exception; end
-end

data/lib/treat/extractors.rb

@@ -1,79 +0,0 @@
-# Extractors extract information out of texts.
-module Treat::Extractors
-
-  # Extracts the language from an entity.
-  module Language
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:entity]
-    self.default = :what_language
-  end
-
-  # Extracts the date/time of a phrase.
-  module Time
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:phrase]
-  end
-
-  # Extract the topic from a document or zone.
-  module Topics
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:document, :section, :zone]
-  end
-
-  # Extract the keywords from a text.
-  module Keywords
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:document, :section, :zone]
-  end
-
-  # Extract clusters of topic words from a collection.
-  module TopicWords
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:collection]
-  end
-
-  # Extract named entities from phrases.
-  module NameTag
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:phrase, :word]
-  end
-
-  # Extract coreferences from a zone.
-  module Coreferences
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:zone]
-  end
-  
-  # Retrieve the main grammatical roles
-  # in the phrase (subject, verb, object).
-  module Roles
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:phrase]
-  end
-
-  module TfIdf
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:word]
-    self.default = :native
-  end
-  
-  module Summary
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:document]
-    self.default = :keyword_count
-  end
-  
-  # Make Extractors categorizable.
-  extend Treat::Categorizable
-  
-end

data/lib/treat/formatters/serializers/mongo.rb

@@ -1,64 +0,0 @@
-# Stores an entity in a Mongo collection.
-class Treat::Formatters::Serializers::Mongo
-
-  # Reauire the Mongo DB
-  require 'mongo'
-
-  # Serialize an entity tree in XML format.
-  #
-  # Options:
-  # - (String) :file => a file to write to.
-  def self.serialize(entity, options = {})
-
-    unless options[:database]
-      raise Treat::Exception,
-      'Must supply the database name.'
-    end
-
-    @@conn ||= Mongo::Connection.new
-    @@db ||= @@conn[options[:database]]
-
-    path = []
-    
-    entity.each_ancestor do |ancestor|
-      path << [ancestor.type, ancestor.id]
-    end
-    
-    path = path.reverse
-    
-    target = @@db
-    
-    path.each do |type_id|
-      coll = @@db[type_id[0]][type_id[1]]
-    end
-    
-    # Store path 
-    
-    Treat::Entities.list.each do |type|
-
-      type = entity.type.to_s
-      type = (type == 'entity') ? 'entities' : (type + 's')
-      doc = coll[type]
-      
-      features = {}
-      features['id'] = entity.id
-      features['value'] = entity.value
-
-      entity.features.each_pair do |feature, value|
-        if value.is_a? Treat::Entities::Entity
-          value = value.id
-        elsif value.is_a?(Array) || value.is_a?(Hash)
-          value = value.inspect
-        else
-          value = value.to_s
-        end
-        features[feature.to_s] = value
-      end
-      
-      doc.insert(features)
-
-    end
-    
-  end
-
-end

data/lib/treat/formatters.rb

@@ -1,41 +0,0 @@
-# Formatters handle conversion of Entities to and from
-# external file formats.
-module Treat::Formatters
-  
-  # Readers read a document's content.
-  module Readers
-    extend Treat::Groupable
-    self.type = :computer
-    self.targets = [:document]
-  end
-  
-  # Unserializers recreate entities 
-  # from a serialized format.
-  module Unserializers
-    extend Treat::Groupable
-    self.type = :computer
-    self.targets = [:entity]
-  end
-  
-  # Serializers transform entities 
-  # into a storable format.
-  module Serializers
-    extend Treat::Groupable
-    self.type = :computer
-    self.targets = [:entity]
-    self.default = :yaml
-  end
-  
-  # Visualizers transform entities 
-  # into a visualizable format.
-  module Visualizers
-    extend Treat::Groupable
-    self.type = :computer
-    self.targets = [:entity]
-    self.default = :tree
-  end
-  
-  # Make Formatters categorizable.
-  extend Treat::Categorizable
-  
-end

data/lib/treat/helpers/decimal_point_escaper.rb

@@ -1,22 +0,0 @@
-module Treat::Helpers
-  
-  class DecimalPointEscaper
-
-    EscapeChar = '^^'
-    EscapedEscapeChar = '\^\^'
-    
-    def self.escape!(s)
-      s.gsub!(/([0-9]+)\.([0-9]+)/) do
-        $1 + EscapeChar + $2
-      end
-    end
-
-    def self.unescape!(s)
-      s.gsub!(/([0-9]+)#{EscapedEscapeChar}([0-9]+)/) do
-        $1 + '.' + $2
-      end
-    end
-
-  end
-  
-end

data/lib/treat/inflectors.rb

@@ -1,52 +0,0 @@
-# Category of worker groups that retrieve
-# the inflections of a word.
-module Treat::Inflectors
-
-  # Return the stem (*not root form*) of a word.
-  module Stemmers
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:word]
-  end
-
-  # Retrieve the different declensions of a
-  # noun (singular, plural).
-  module Declensors
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:word]
-    self.preset_option = :count
-    self.presets = [:plural, :singular]
-  end
-
-  # Retrieve the different conjugations of a word
-  # given a mode, tense, person, and/or number.
-  module Conjugators
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:word]
-    self.preset_option = :form
-    self.presets = [:infinitive, :present_participle, 
-                    :plural_verb, :singular_verb]
-  end
-
-  # Retrieve the full text description of a
-  # cardinal number.
-  module Cardinalizers
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:number]
-  end
-
-  # Retrieve the full text description of an
-  # ordinal number.
-  module Ordinalizers
-    extend Treat::Groupable
-    self.type = :annotator
-    self.targets = [:number]
-  end
-
-  # Make Inflectors categorizable.
-  extend Treat::Categorizable
-
-end

data/lib/treat/kernel.rb

@@ -1,208 +0,0 @@
-# Extends the core Kernel module to provide
-# easy access to utility functions used across
-# the library.
-module Kernel
-  
-  # Require file utilities for creating and
-  # deleting temporary files.
-  require 'fileutils'
-
-  # A list of acronyms used in class names within
-  # the program. These do not CamelCase; they
-  # CAMELCase.
-  Acronyms = %w[xml html txt odt abw doc yaml uea lda pdf ptb dot ai id3 svo mlp]
-
-  # A cache to optimize camel casing.
-  @@cc_cache = {}
-
-  # A cache to optimize un camel casing.
-  @@ucc_cache = {}
-
-  # Runs a block of code without warnings.
-  def silence_warnings(&block)
-    warn_level = $VERBOSE
-    $VERBOSE = nil
-    result = block.call
-    $VERBOSE = warn_level
-    result
-  end
-
-  # Runs a block of code while blocking stdout.
-  def silence_stdout(log = NULL_DEVICE)
-    unless Treat.silence
-      yield; return
-    end
-    old = $stdout.dup
-    $stdout.reopen(File.new(log, 'w'))
-    yield
-    $stdout = old
-  end
-  
-  # Create a temporary file which is deleted
-  # after execution of the block.
-  def create_temp_file(ext, value = nil, &block)
-    fname = Treat.tmp + 
-    "#{Random.rand(10000000).to_s}.#{ext}"
-    File.open(fname, 'w') do |f|
-      f.write(value) if value
-      block.call(f.path)
-    end
-  ensure
-    File.delete(fname)
-  end
-
-  # Create a temporary directory, which is
-  # deleted after execution of the block.
-  def create_temp_dir(&block)
-    dname = "#{Treat.lib}/../tmp/"+
-    "#{Random.rand(10000000).to_s}"
-    Dir.mkdir(dname)
-    block.call(dname)
-  ensure
-    FileUtils.rm_rf(dname)
-  end
-
-  # Convert un_camel_case to CamelCase.
-  def camel_case(o_phrase)
-    phrase = o_phrase.to_s.dup
-    return @@cc_cache[o_phrase] if @@cc_cache[o_phrase]
-
-    if Acronyms.include?(phrase)
-      phrase = phrase.upcase
-    else
-      phrase.gsub!(/^[a-z]|_[a-z]/) { |a| a.upcase }
-      phrase.gsub!('_', '')
-    end
-    @@cc_cache[o_phrase] = phrase
-  end
-
-  alias :cc :camel_case
-
-  # Convert CamelCase to un_camel_case.
-  def un_camel_case(o_phrase)
-    phrase = o_phrase.to_s.dup
-    return @@ucc_cache[o_phrase] if @@ucc_cache[o_phrase]
-    if Acronyms.include?(phrase.downcase)
-      phrase = phrase.downcase
-    else
-      phrase.gsub!(/[A-Z]/) { |p| '_' + p.downcase  }
-      phrase = phrase[1..-1] if phrase[0] == '_'
-    end
-    @@ucc_cache[o_phrase] = phrase
-  end
-
-  alias :ucc :un_camel_case
-
-  # Retrieve the Class from a Module::Class.
-  def class_name(n); n.to_s.split('::')[-1]; end
-
-  alias :cl :class_name
-
-  # Search the list to see if there are words similar to #name
-  # in the #list If yes, return a string saying "Did you mean
-  # ... ?" with the names.
-  def did_you_mean?(list, name)
-    return '' # Fix
-    list = list.map { |e| e.to_s }
-    name = name.to_s
-    sugg = []
-    list.each do |element|
-      l = levenshtein(element,name)
-      if  l > 0 && l < 2
-        sugg << element
-      end
-    end
-    unless sugg.size == 0
-      if sugg.size == 1
-        msg += " Perhaps you meant '#{sugg[0]}' ?"
-      else
-        sugg_quote = sugg[0..-2].map do
-          |x| '\'' + x + '\''
-        end
-        msg += " Perhaps you meant " +
-        "#{sugg_quote.join(', ')}," +
-        " or '#{sugg[-1]}' ?"
-      end
-    end
-    msg
-  end
-
-  alias :dym? :did_you_mean?
-
-  # Return the name of the method that called the method
-  # that calls this method.
-  def caller_method(n = 3)
-    at = caller(n).first
-    /^(.+?):(\d+)(?::in `(.*)')?/ =~ at
-    Regexp.last_match[3].gsub('block in ', '').intern
-  end
-
-  alias :cm :caller_method
-
-  # Detect the platform we're running on.
-  def detect_platform
-    p = RUBY_PLATFORM.downcase
-    return :mac if p.include?("darwin")
-    return :windows if p.include?("mswin")
-    return :linux if p.include?("linux")
-    return :unknown
-  end
-
-  # Return the levensthein distance between two stringsm
-  # taking into account the costs of insertion, deletion,
-  # and substitution. Stolen from:
-  # http://ruby-snippets.heroku.com/string/levenshtein-distance
-  # Used by did_you_mean?
-  def levenshtein(first, other, ins=1, del=1, sub=1)
-    return nil if first.nil? || other.nil?
-    dm = []
-    dm[0] = (0..first.length).collect { |i| i * ins}
-    fill = [0] * (first.length - 1).abs
-    for i in 1..other.length
-      dm[i] = [i * del, fill.flatten]
-    end
-    for i in 1..other.length
-      for j in 1..first.length
-        dm[i][j] = [
-          dm[i-1][j-1] +
-          (first[i-1] ==
-          other[i-1] ? 0 : sub),
-          dm[i][j-1] + ins,
-          dm[i-1][j] + del
-        ].min
-      end
-    end
-    dm[other.length][first.length]
-  end
-
-  if detect_platform == :windows
-    NULL_DEVICE = 'NUL'
-  else
-    NULL_DEVICE = '/dev/null'
-  end
-  
-  def prompt(msg, valid_answers)
-    
-    msg = msg
-    n = msg.include?("\n") ? ":\n" : ''
-    q = msg.include?("\n") ? '' : '?'
-    
-    s = "\nPlease enter one of #{valid_answers.join(', ')}: "
-    puts "Do you want to #{n}#{msg}#{q} \n#{s}"
-    
-    begin
-      answer = STDIN.gets.strip
-      unless valid_answers.include?(answer)
-        puts "Invalid input."
-        puts s
-        raise Treat::InvalidInputException
-      end
-      puts
-      answer
-    rescue Treat::InvalidInputException
-      retry
-    end
-    
-  end
-
-end

data/lib/treat/languages/arabic.rb

@@ -1,16 +0,0 @@
-class Treat::Languages::Arabic
-  
-  RequiredDependencies = []
-  OptionalDependencies = []
-  
-  Extractors = {}
-  Inflectors = {}
-  Lexicalizers = {
-    :taggers => [:stanford]
-  }
-  Processors = {
-    :parsers => [:stanford]
-  }
-  Retrievers = {}
-  
-end

data/lib/treat/languages/chinese.rb

@@ -1,16 +0,0 @@
-class Treat::Languages::Chinese
-  
-  RequiredDependencies = []
-  OptionalDependencies = []
-  
-  Extractors = {}
-  Inflectors = {}
-  Lexicalizers = {
-    :taggers => [:stanford]
-  }
-  Processors = {
-    :parsers => [:stanford]
-  }
-  Retrievers = {}
-  
-end