treat 0.2.5 → 1.0.0

data/lib/treat/configurable.rb

@@ -0,0 +1,115 @@
+# This module provides configuration options for the Treat toolkit
+# (enable/disable syntactic sugar, enable/disable language detection
+# and set default language or language detection level.
+module Treat::Configurable
+
+  # Modify the singleton class of the base module (Treat).
+  def self.extended(base)
+
+    # Configuration options that are available for the Treat module.
+    class << base
+      # Symbol - default language to use when detect_language is false.
+      attr_accessor :default_language
+      # Boolean - detect language or use default?
+      attr_accessor :detect_language
+      # Symbol - the finest entity level at which to detect language.
+      attr_accessor :language_detection_level
+      # Boolean - whether to output debug information or not.
+      attr_accessor :debug
+      # Boolean - whether to silence the output of external programs.
+      attr_accessor :silence
+    end
+
+    # Set the default options.
+    base.module_eval do
+      # Set the default language to english.
+      self.default_language = :eng
+      # Turn language detection off by default.
+      self.detect_language = false
+      # Detect the language once per document by default.
+      self.language_detection_level = :document
+      # Set debug to off by default.
+      self.debug = false
+      # Silence external programs by default.
+      self.silence = true
+    end
+
+  end
+
+  # Turn on syntactic sugar for the creation of Entities.
+  #
+  # All entities found under Treat::Entities will be made
+  # available within the global namespace. As an example,
+  # 'Treat::Entities::Word' can then be referred to as 'Word'.
+  #
+  # There is one exception: the Symbol class is not sweetened
+  # to avoid clashing with the Symbol class defined by Ruby.
+  def sweeten!
+    return if @@sweetened
+    @@sweetened = true
+    each_entity_class do |type, klass|
+      Object.class_eval do
+        unless type == :Symbol
+          define_method(type) do |file_or_value, options={}|
+            klass.build(file_or_value, options)
+          end
+        end
+      end
+    end
+  end
+
+  # Turn off syntactic sugar.
+  def unsweeten!
+    return unless @@sweetened
+    @@sweetened = false
+    each_entity_class do |type, klass|
+      Object.class_eval do
+        remove_method(type)
+      end unless type == :Symbol
+    end
+  end
+
+  # Boolean - whether syntactic sugar is
+  # enabled or not.
+  def sweetened?; @@sweetened; end
+
+  # Syntactic sugar is disabled by default.
+  @@sweetened = false
+
+  # Turn on language detection, optionally setting
+  # the language detection level (finest level at
+  # which language detection is performed).
+  def self.detect!(level = nil)
+    self.detect_language = true
+    if level
+      self.language_detection_level = level
+    end
+  end
+
+  # Turn off language detection, optionally setting
+  # a new default language to use.
+  def self.undetect!(default = :english)
+    self.detect_language = false
+    if default
+      self.default_language = default
+    end
+  end
+
+  # Use the supplied language by default and
+  # turn off language detection.
+  def self.use(language)
+    self.detect_language = false
+    self.default_language = language
+  end
+  
+  private
+  # Helper method, yields each entity type and class.
+  def each_entity_class
+    Treat::Entities.list.each do |entity_type|
+      type = cc(entity_type).intern
+      klass = Treat::Entities.const_get(type, klass)
+      yield type, klass
+    end
+  end
+
+end

data/lib/treat/data_set.rb

@@ -0,0 +1,42 @@
+class Treat::DataSet
+
+  require 'psych'
+  require 'treat/classification'
+  
+  attr_reader :classification
+  attr_reader :labels
+  attr_reader :items
+  attr_reader :ids
+  
+  def self.open(file)
+    unless File.readable?(file)
+      raise Treat::Exception,
+      "Cannot load data set "+
+      "from #{file} because " +
+      "it doesn't exist."
+    end
+    ::Psych.load(
+    File.read(file))
+  end
+  
+  def initialize(classification)
+    @classification = classification
+    @labels = classification.labels
+    @items = []
+    @ids = []
+  end
+  
+  def <<(entity)
+    @items << 
+    @classification.
+    export_item(entity)
+    @ids << entity.id
+  end
+  
+  def save(file)
+    File.open(file, 'w') do |f|
+      f.write(::Psych.dump(self))
+    end
+  end
+  
+end

data/lib/treat/dependencies.rb

@@ -0,0 +1,24 @@
+class Treat::Dependencies
+  
+  Gem = [
+    ['psych', '1.2.2', '(un)serialize annotated entities to YAML format'],
+    ['nokogiri', '>= 1.4.0', 'read and parse XML and HTML formats'],
+    ['sdsykes-ferret', '>= 0.11.6.19', 'perform full-text search in collections'],
+    ['lda-ruby', '>= 0.3.8', 'extract topic words from documents and collections'],
+    ['ruby-readability', '>= 0.5.0', 'extract the readable content from HTML pages'],
+    ['stanford-core-nlp', '>= 0.1.8', 'tokenize, segment, parse texts and perform named entity recognition'],
+    ['whatlanguage', '>= 1.0.0', 'detect the language of text'],
+    ['linguistics', '>= 1.0.9', 'retrieve the inflection of nouns, verbs and numbers in English'],
+    ['punkt-segmenter', '>= 0.9.1', 'segment texts into sentences'],
+    ['chronic', '>= 0.6.7', 'detect date and time in text'],
+    ['decisiontree', '>= 0.3.0', 'perform decision tree classification of text entities']
+  ]
+  
+  Binary = [
+    ['ocropus', 'recognize text in image files'],
+    ['antiword', 'extract text from DOC files'],
+    ['poppler-utils', 'extract text from PDF files'],
+    ['graphviz', 'export and visualize directed graphs']
+  ]
+
+end

data/lib/treat/downloader.rb

@@ -0,0 +1,87 @@
+# Download a file without storing it entirely in memory.
+class Treat::Downloader
+
+  require 'net/http'
+  require 'fileutils'
+
+  class << self
+    attr_accessor :show_progress
+  end
+
+  self.show_progress = false
+
+  MaxTries = 3
+  
+  # Download a file into destination, and return
+  # the path to the downloaded file. If the filename 
+  # is nil, it will set the default filename to 'top'.
+  def self.download(protocol, server, dir, file = nil, target_base = nil, target_dir = nil)
+
+    require 'progressbar' if self.show_progress
+    
+    target_base ||= Treat.files
+    target_dir ||= server
+   
+    dir += '/' if dir && dir[-1] != '/'
+    resource = "#{dir}#{file}"
+    resource = "/#{resource}" unless resource[0] == '/'
+    url = "#{server}#{resource}"
+    path = File.join(target_base, target_dir)
+    
+    unless FileTest.directory?(path)
+      FileUtils.mkdir(path)
+    end
+    
+    
+    file = File.open("#{path}/#{file}", 'w')
+    tries = 0
+    begin
+
+      Net::HTTP.start(server) do |http|
+
+        http.use_ssl = true if protocol == 'https'
+        
+        http.request_get(resource) do |response|
+
+          if response.content_length
+            length = response.content_length
+          else
+            warn 'Unknown file size; ETR unknown.'
+            length = 10000
+          end
+
+          pbar = self.show_progress ?
+          ProgressBar.new(url, length)  : nil
+
+          unless response.code == '200'
+            raise Treat::Exception,
+            "response code was not 200 "+
+            "OK, but was #{response.code}. "
+          end
+
+          response.read_body do |segment|
+            pbar.inc(segment.length) if pbar
+            file.write(segment)
+          end
+
+          pbar.finish if pbar
+
+        end
+
+      end
+
+      file.path.to_s
+
+    rescue Exception => error
+      tries += 1
+      retry if tries < MaxTries
+      raise Treat::Exception,
+      "Couldn't download #{url}. (#{error.message})"
+      file.delete
+    ensure
+      file.close
+    end
+
+  end
+
+end

data/lib/treat/entities.rb

@@ -1,74 +1,76 @@
-module Treat
-  # Abstract and concrete structures extending the
-  # Tree::Node class to represent textual entities:
-  #
-  # - Collection
-  # - Document
-  # - Zone (a Section, Title, Paragraph, or List)
-  # - Sentence
-  # - Phrases
-  # - Token (a Word, Number, Punctuation, or Symbol).
-  module Entities
-    # Cache a list of defined entity types to
-    # improve performance.
-    @@list = nil
-    # Provide a list of defined entity types,
-    # as non-camel case identifiers.
-    def self.list
-      return @@list if @@list
-      @@list = []
-      self.constants.each do |constant|
-        unless constant == :Entity
-          @@list << ucc(constant).intern 
-        end
-      end
-      @@list
+# Entities are Tree structures that represent textual entities
+# (from a collection of texts down to an individual word) with
+# a unique identifier, a value, features, children and dependencies
+# linking them to other textual entities.
+#
+# - A Collection represents a folder containing documents (and folders).
+# - A Document represents a file with a textual content.
+# - A Zone represents a logical division of content in a document.
+# - A Phrase is a group of words; a Sentence is a Phrase with an ender.
+# - A Token represents a Word, a Number, a Punctuation or a Symbol.
+module Treat::Entities
+
+  # Variables for the singleton class.
+  class << self
+    # Provide a list of all entity types except Entity,
+    # as non_camel_case identifiers.
+    attr_accessor :list
+  end
+
+  # Require all entities.
+  require 'treat/entities/entities'
+  
+  # Add each constant to the list, except Entity.
+  self.list = []
+  constants.each do |constant|
+    unless constant == :Entity || 
+           constant == :Abilities
+      self.list << ucc(constant).intern
     end
-    # Require Entity first.
-    require 'treat/entities/entity'
-    # Then require all possible entities.
-    require 'treat/entities/collection'
-    require 'treat/entities/document'
-    require 'treat/entities/zones'
-    require 'treat/entities/phrases'
-    require 'treat/entities/tokens'
-    # Make the constants buildable.
-    constants.each do |entity|
-      define_singleton_method(entity) do |value='', id=nil|
-        const_get(entity).build(value, id)
-      end
+  end
+
+  # Make each Entity class buildable magically.
+  # This enables to create Entities without calling
+  # #new (e.g. Word 'hello').
+  constants.each do |entity|
+    define_singleton_method(entity) do |value='', id=nil|
+      const_get(entity).build(value, id)
     end
-    # Create entity lookup table.
-    # Lookup table
-    @@match_types = nil
-    def self.match_types
-      return @@match_types if @@match_types
-      list = (Treat::Entities.list + [:entity])
-      @@match_types = {}
-      list.each do |type1|
-        list.each do |type2|
-          @@match_types[type2] ||= {}
-          if (type1 == type2) || 
-            (Treat::Entities.const_get(cc(type1)) < 
-            Treat::Entities.const_get(cc(type2)))
-            @@match_types[type2][type1] = true
-          end
+  end
+
+  # Create entity lookup table.
+  @@match_types = nil
+  def self.match_types
+    return @@match_types if @@match_types
+    list = (Treat::Entities.list + [:entity])
+    @@match_types = {}
+    list.each do |type1|
+      list.each do |type2|
+        @@match_types[type2] ||= {}
+        if (type1 == type2) ||
+          (Treat::Entities.const_get(cc(type1)) <
+          Treat::Entities.const_get(cc(type2)))
+          @@match_types[type2][type1] = true
         end
       end
-      @@match_types
     end
-    # Return the hierarchy level of the entity
-    # class, the minimum being a Token and the
-    # maximum being a Collection.
-    def self.rank(type)
-      klass = Entities.const_get(cc(type))
-      compare = lambda { |a,b| a == b || a < b }
-      return 0 if compare.call(klass, Token)
-      return 1 if compare.call(klass, Phrase)
-      return 2 if compare.call(klass, Sentence)
-      return 3 if compare.call(klass, Zone)
-      return 4 if compare.call(klass, Document)
-      return 5 if compare.call(klass, Collection)
+    @@match_types
+  end
+  
+  # A bottom-up ordering of general types of entities.
+  @@order = [Token, Phrase, Zone, Document, Collection]
+  
+  # Return the hierarchy level of the entity
+  # class, the minimum being a Token and the
+  # maximum being a Collection.
+  #
+  # Implement as true comparison functions.
+  def self.rank(type)
+    klass = Treat::Entities.const_get(cc(type))
+    compare = lambda { |a,b| a == b || a < b }
+    1.upto(@@order.size) do |i|
+      return i if compare.call(klass, @@order[i])
     end
   end
+
 end

data/lib/treat/entities/abilities.rb

@@ -0,0 +1,10 @@
+# Requires the -able mixins for the Entity class.
+module Treat::Entities::Abilities
+  
+  p = 'treat/entities/abilities/*.rb'
+  
+  Dir[Treat.lib + p].each do |f| 
+    require f
+  end
+  
+end

data/lib/treat/entities/abilities/buildable.rb

@@ -0,0 +1,327 @@
+# Represents an object that can be built
+# from a folder of files, a specific file,
+# a string or a numeric object. This class
+# is pretty much self-explanatory.
+module Treat::Entities::Abilities::Buildable
+
+  require 'treat/helpers/decimal_point_escaper'
+  require 'fileutils'
+
+  # Simple regexps to match common entities.
+  WordRegexp = /^[[:alpha:]\-']+$/
+  NumberRegexp = /^#?([0-9]+)(\^\^[0-9]+)?$/
+  PunctRegexp = /^[[:punct:]\$]+$/
+  UriRegexp = /^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(([0-9]{1,5})?\/.*)?$/ix
+  EmailRegexp = /.+\@.+\..+/
+
+  # Reserved folder names
+  Reserved = ['.index']
+
+  # Build an entity from anything (can be
+  # a string, numeric,folder, or file name
+  # representing a raw or serialized file).
+  def build(file_or_value, options = {})
+
+    fv = file_or_value.to_s
+    if self == Treat::Entities::Document
+      if fv =~ UriRegexp
+        from_url(fv, options)
+      else
+        from_file(fv, options)
+      end
+    elsif self == Treat::Entities::Collection
+      if FileTest.directory?(fv)
+        from_folder(fv, options)
+      else
+        create_collection(fv)
+      end
+    else
+      if file_or_value.is_a?(String)
+        from_string(file_or_value)
+      elsif file_or_value.is_a?(Numeric)
+        from_numeric(file_or_value)
+      else
+        raise Treat::Exception,
+        "Unrecognizable input '#{fv}'. "+
+        "Please supply a folder, " +
+        "filename, string or number."
+      end
+    end
+    
+  end
+
+  # Build an entity from a string. Type is
+  # enforced only if requested or if the entity
+  # is user-created (i.e. by calling build
+  # instead of from_string directly).
+  def from_string(string, enforce_type = false)
+
+    Treat::Helpers::DecimalPointEscaper.escape!(string)
+
+    enforce_type = true if caller_method == :build
+
+    unless self == Treat::Entities::Entity
+      return self.new(string) if enforce_type
+    end
+
+    e = anything_from_string(string)
+
+    if enforce_type && !e.is_a?(self)
+      raise "Asked to build a #{cl(self).downcase} "+
+      "from \"#{string}\" and to enforce type, "+
+      "but type detected was #{cl(e.class).downcase}."
+    end
+
+    e
+  end
+
+  # Build a document from an URL.
+  def from_url(url, options)
+    unless self ==
+      Treat::Entities::Document
+      raise Treat::Exception,
+      'Cannot create something ' +
+      'else than a document from a url.'
+    end
+
+    uri = ::URI.parse(url)
+
+    sp = uri.path.split('/')
+    sp.shift if sp[0] == ''
+
+    file = sp[-1]
+    path = sp.size == 1 ?
+    '/' : sp[0..-2].join('/')
+
+    f = Treat::Downloader.download(
+    uri.scheme, uri.host, path, file)
+    options[:_default_format] = :html
+
+    e = from_file(f, options)
+    e.set :url, url
+    e
+
+  end
+
+  # Build an entity from a Numeric object.
+  def from_numeric(numeric)
+    unless (self ==
+      Treat::Entities::Number) ||
+      (self == Treat::Entities::Token) ||
+      (self == Treat::Entities::Entity)
+      raise Treat::Exception,
+      "Cannot create something " +
+      "else than a number/token from " +
+      "a numeric object."
+    end
+    n = numeric.to_s
+    Treat::Helpers::DecimalPointEscaper.unescape!(n)
+    Treat::Entities::Number.new(n)
+  end
+
+  # Build an entity from a folder with documents.
+  # Folders will be searched recursively.
+  def from_folder(folder, options)
+
+    return if Reserved.include?(folder)
+    
+    unless FileTest.directory?(folder)
+      raise Treat::Exception,
+      "Path '#{folder}' does " +
+      "not point to a folder."
+    end
+
+    unless File.readable?(folder)
+      raise Treat::Exception,
+      "Folder '#{folder}' is not readable."
+    end
+
+    unless self ==
+      Treat::Entities::Collection
+      raise Treat::Exception,
+      "Cannot create something " +
+      "else than a collection " +
+      "from folder '#{folder}'."
+    end
+
+    c = Treat::Entities::Collection.new(folder)
+    folder += '/' unless folder[-1] == '/'
+
+    Dir[folder + '*'].each do |f|
+      if FileTest.directory?(f)
+        c2 = Treat::Entities::Collection.
+        from_folder(f, options)
+        c.<<(c2, false) if c2
+      else
+        c.<<(Treat::Entities::Document.
+        from_file(f, options), false)
+      end
+    end
+    c
+
+  end
+
+  # Build a document from a raw or serialized file.
+  def from_file(file, options)
+    
+    unless File.readable?(file)
+      raise Treat::Exception,
+      "Path '#{file}' does not "+
+      "point to a readable file."
+    end
+
+    dflt = options[:_default_format]
+    fmt = Treat::Formatters::Readers::Autoselect.
+    detect_format(file, dflt)
+    options[:_format] = fmt
+    
+    if fmt == :yaml || fmt == :yml ||
+      (fmt == :xml && is_treat_xml?(file))
+      f = from_serialized_file(file, options)
+    else
+      f = from_raw_file(file, options)
+    end
+
+  end
+
+  # Build a document from a raw file.
+  def from_raw_file(file, options)
+
+    unless self ==
+      Treat::Entities::Document
+      raise Treat::Exception,
+      "Cannot create something else than a " +
+      "document from raw file '#{file}'."
+    end
+
+    d = Treat::Entities::Document.new(file)
+
+    d.read(:autoselect, options)
+
+  end
+
+  # Build an entity from a serialized file.
+  def from_serialized_file(file, options)
+
+    d = Treat::Entities::Document.new(file)
+    d.unserialize(:autoselect, options)
+    d.children[0].set_as_root!
+    d.children[0]
+
+  end
+
+  # Build any kind of entity from a string.
+  def anything_from_string(string)
+
+    case cl(self).downcase.intern
+    when :document, :collection
+      raise Treat::Exception,
+      "Cannot create a document or " +
+      "collection from a string " +
+      "(need a readable file/folder)."
+    when :phrase
+      phrase_from_string(string)
+    when :token
+      token_from_string(string)
+    when :zone
+      zone_from_string(string)
+    when :entity
+      if string.count(' ') == 0
+        token_from_string(string)
+      else
+        if string.gsub(/[\.\!\?]+/,
+          '.').count('.') <= 1 &&
+          string.count("\n") == 0
+          phrase_from_string(string)
+        else
+          zone_from_string(string)
+        end
+      end
+    else
+      self.new(string)
+    end
+
+  end
+
+  def check_encoding(string)
+    string.encode("UTF-8", undef: :replace) # Fix
+  end
+
+  # Build a phrase from a string.
+  def phrase_from_string(string)
+
+    check_encoding(string)
+
+    if string.count('.!?') >= 1
+      Treat::Entities::Sentence.new(string)
+    else
+      Treat::Entities::Phrase.new(string)
+    end
+
+  end
+
+  # Build the right type of token
+  # corresponding to a string.
+  def token_from_string(string)
+
+    check_encoding(string)
+    if string == "'s" || string == "'S"
+      Treat::Entities::Clitic.new(string)
+    elsif string =~ WordRegexp &&
+      string.count(' ') == 0 &&
+      string != '-'
+      Treat::Entities::Word.new(string)
+    elsif string =~ NumberRegexp
+      from_numeric(string)
+    elsif string =~ PunctRegexp
+      Treat::Entities::Punctuation.new(string)
+    elsif string.count('.') > 0 &&
+      string =~ UriRegexp
+      Treat::Entities::Url.new(string)
+    elsif string.count('@') > 0 &&
+      string =~ EmailRegexp
+      Treat::Entities::Email.new(string)
+    else
+      Treat::Entities::Symbol.new(string)
+    end
+  end
+
+  # Build the right type of zone
+  # corresponding to the string.
+
+  def zone_from_string(string)
+
+    check_encoding(string)
+    dot = string.count('.!?')
+    if dot && dot >= 1 && string.count("\n") > 0
+      Treat::Entities::Section.new(string)
+    elsif string.count('.') == 0 &&
+      string.size < 45
+      Treat::Entities::Title.new(string)
+    else
+      Treat::Entities::Paragraph.new(string)
+    end
+
+  end
+  
+  # Eventually find a better way.
+  def is_treat_xml?(file)
+
+    beginning = nil
+
+    File.open(file) do |w|
+      beginning = w.readlines(200)
+    end
+
+    beginning = beginning.join(' ')
+    beginning.count('<treat>') > 0
+
+  end
+
+  def create_collection(fv)
+    debug("Creating new collection in directory #{fv}.")
+    FileUtils.mkdir(fv)
+    Treat::Entities::Collection.new(fv)
+  end
+
+end