treat 1.0.6 → 1.1.0

data/lib/treat/config/tags/enju.rb

@@ -0,0 +1,71 @@
+{cat_to_category: {
+  'ADJ' => 'adjective',
+  'ADV' => 'adverb',
+  'CONJ' => 'conjunction',
+  'COOD' => 'conjunction',
+  'C' => 'complementizer',
+  'D' => 'determiner',
+  'N' => 'noun',
+  'P' => 'preposition',
+  'PN' => 'punctuation',
+  'SC' => 'conjunction',
+  'V' => 'verb',
+  'PRT' => 'particle'
+},
+cat_to_description: [
+  ['ADJ',	'Adjective'],
+  ['ADV',	'Adverb'],
+  ['CONJ',	'Coordination conjunction'],
+  ['C',	'Complementizer'],
+  ['D',	'Determiner'],
+  ['N',	'Noun'],
+  ['P',	'Preposition'],
+  ['SC',	'Subordination conjunction'],
+  ['V',	'Verb'],
+  ['COOD',	'Part of coordination'],
+  ['PN',	'Punctuation'],
+  ['PRT',	'Particle'],
+  ['S',	'Sentence']  
+],
+xcat_to_description: [
+  ['COOD',	'Coordinated phrase/clause'],
+  ['IMP',	'Imperative sentence'],
+  ['INV',	'Subject-verb inversion'],
+  ['Q',	'Interrogative sentence with subject-verb inversion'],
+  ['REL',	'A relativizer included'],
+  ['FREL', 'A free relative included'],
+  ['TRACE',	'A trace included'],
+  ['WH', 'A wh-question word included']
+],
+xcat_to_ptb: [
+  ['ADJP', '', 'ADJP'],
+  ['ADJP', 'REL', 'WHADJP'],
+  ['ADJP', 'FREL', 'WHADJP'],
+  ['ADJP', 'WH', 'WHADJP'],
+  ['ADVP', '', 'ADVP'],
+  ['ADVP', 'REL', 'WHADVP'],
+  ['ADVP', 'FREL', 'WHADVP'],
+  ['ADVP', 'WH', 'WHADVP'],
+  ['CONJP', '', 'CONJP'],
+  ['CP', '', 'SBAR'],
+  ['DP', '', 'NP'],
+  ['NP', '', 'NP'],
+  ['NX', 'NX', 'NAC'],
+  ['NP'	'REL'	'WHNP'],
+  ['NP'	'FREL'	'WHNP'],
+  ['NP'	'WH'	'WHNP'],
+  ['PP', '', 'PP'],
+  ['PP', 'REL', 'WHPP'],
+  ['PP', 'WH', 'WHPP'],
+  ['PRT', '', 'PRT'],
+  ['S', '', 'S'],
+  ['S', 'INV', 'SINV'],
+  ['S', 'Q', 'SQ'],
+  ['S', 'REL', 'SBAR'],
+  ['S', 'FREL', 'SBAR'],
+  ['S', 'WH', 'SBARQ'],
+  ['SCP', '', 'SBAR'],
+  ['VP', '', 'VP'],
+  ['VP', '', 'VP'],
+  ['', '', 'UK']
+]}

data/lib/treat/config/tags/paris7.rb

@@ -0,0 +1,17 @@
+{tag_to_category: {
+  'C' => :complementizer,
+  'PN' => :punctuation,
+  'SC' => :conjunction
+}
+# Paris7 Treebank functional tags
+=begin
+SUJ (subject)
+OBJ (direct object)
+ATS (predicative complement of a subject)
+ATO (predicative complement of a direct object)
+MOD (modifier or adjunct)
+A-OBJ (indirect complement introduced by à)
+DE-OBJ (indirect complement introduced by de)
+P-OBJ (indirect complement introduced by another preposition)
+=end
+}

data/lib/treat/config/tags/ptb.rb

@@ -0,0 +1,15 @@
+{escape_characters: {
+  '(' => '-LRB-',
+  ')' => '-RRB-',
+  '[' => '-LSB-',
+  ']' => '-RSB-',
+  '{' => '-LCB-',
+  '}' => '-RCB-'
+},
+phrase_tag_to_description: [
+  ['S', 'Paris7 declarative clause'],
+  ['SBAR', 'Clause introduced by a (possibly empty) subordinating conjunction'],
+  ['SBARQ', 'Direct question introduced by a wh-word or a wh-phrase'],
+  ['SINV', 'Inverted declarative sentence'],
+  ['SQ', 'Inverted yes/no question']
+]}

data/lib/treat/config/workers/extractors.rb

@@ -0,0 +1,39 @@
+{
+  language: {
+    type: :annotator,
+    targets: [:entity],
+    default: :what_language
+  },
+  time: {
+    type: :annotator,
+    targets: [:phrase]
+  },
+  topics: {
+    type: :annotator,
+    targets: [:document, :section, :zone]
+  },
+  keywords: {
+    type: :annotator,
+    targets: [:document, :section, :zone]
+  },
+  topic_words: {
+    type: :annotator,
+    targets: [:collection]
+  },
+  name_tag: {
+    type: :annotator,
+    targets: [:phrase, :word]
+  },
+  coreferences: {
+    type: :annotator,
+    targets: [:zone]
+  },
+  tf_idf: {
+    type: :annotator,
+    targets: [:word]
+  },
+  summary: {
+    type: :annotator,
+    targets: [:document]
+  }
+}

data/lib/treat/config/workers/formatters.rb

@@ -0,0 +1,20 @@
+{
+  readers: {
+    type: :computer,
+    targets: [:document],
+  },
+  unserializers: {
+    type: :computer,
+    targets: [:entity],
+  },
+  serializers: {
+    type: :computer,
+    targets: [:entity],
+    default: :yaml,
+  },
+  visualizers: {
+    type: :computer,
+    targets: [:entity],
+    default: :tree
+  }
+}

data/lib/treat/config/workers/inflectors.rb

@@ -0,0 +1,27 @@
+{
+  stemmers: {
+    type: :annotator,
+    targets: [:word]
+  },
+  declensors: {
+    type: :annotator,
+    targets: [:word],
+    preset_option: :count,
+    presets: [:plural, :singular]
+  },
+  conjugators: {
+    type: :annotator,
+    targets: [:word],
+    preset_option: :form,
+    presets: [:infinitive, :present_participle, 
+              :plural_verb, :singular_verb]
+  },
+  cardinalizers: {
+    type: :annotator,
+    targets: [:number]
+  },
+  ordinalizers: {
+    type: :annotator,
+    targets: [:number]
+  }
+}

data/lib/treat/config/workers/learners.rb

@@ -0,0 +1,6 @@
+{
+  classifiers: {
+    type: :computer,
+    targets: [:entity]
+  }
+}

data/lib/treat/config/workers/lexicalizers.rb

@@ -0,0 +1,18 @@
+{
+  taggers: {
+    type: :annotator,
+    targets: [:phrase, :token]
+  },
+  categorizers: {
+    type: :annotator,
+    targets: [:phrase, :token],
+    recursive: true
+  },
+  sensers: {
+    type: :annotator,
+    targets: [:word],
+    preset_option: :nym,
+    presets: [:synonyms, :antonyms, 
+              :hyponyms, :hypernyms],
+  }        
+}

data/lib/treat/config/workers/list.rb

@@ -0,0 +1 @@
+[:extractors, :inflectors, :formatters, :learners, :lexicalizers, :processors, :retrievers]

data/lib/treat/config/workers/processors.rb

@@ -0,0 +1,19 @@
+{
+  chunkers: {
+    type: :transformer,
+    targets: [:document],
+    default: :autoselect
+  },
+  segmenters: {
+    type: :transformer,
+    targets: [:zone]
+  },
+  tokenizers: {
+    type: :transformer,
+    targets: [:sentence, :phrase]
+  },
+  parsers: {
+    type: :transformer,
+    targets: [:sentence, :phrase] 
+  }
+}

data/lib/treat/config/workers/retrievers.rb

@@ -0,0 +1,12 @@
+{
+  indexers: {
+    type: :annotator,
+    targets: [:collection],
+    default: :ferret
+  },
+  searchers: {
+    type: :computer,
+    targets: [:collection],
+    default: :ferret
+  }
+}

data/lib/treat/config.rb

@@ -0,0 +1,125 @@
+module Treat::Config
+
+  Paths = [ :tmp, :lib, :bin, 
+  :files, :data, :models, :spec ]
+
+  class << self
+    attr_accessor :config
+  end
+
+  Treat.module_eval do
+    # Handle all missing methods as conf options.
+    def self.method_missing(sym, *args, &block)
+      super(sym, *args, &block) if sym == :to_ary
+      Treat::Config.config[sym]
+    end
+  end
+
+  def self.configure
+    # Temporary configuration hash.
+    config = { paths: {} }
+    confdir = get_full_path(:lib) + 'treat/config'
+    # Iterate over each directory in the config.
+    Dir[confdir + '/*'].each do |dir|
+      name = File.basename(dir, '.*').intern
+      config[name] = {}
+      # Iterate over each file in the directory.
+      Dir[confdir + "/#{name}/*.rb"].each do |file|
+        key = File.basename(file, '.*').intern
+        config[name][key] = eval(File.read(file))
+      end
+    end
+    # Get the path config.
+    Paths.each do |path|
+      config[:paths][path] = get_full_path(path)
+    end
+    # Get the tag alignments.
+    configure_tags!(config[:tags][:aligned])
+    # Convert hash to structs.
+    self.config = self.hash_to_struct(config)
+  end
+
+  def self.get_full_path(dir)
+    File.dirname(__FILE__) +
+    '/../../' + dir.to_s + "/"
+  end
+  
+  def self.configure_tags!(config)
+    ts = config[:tag_sets]
+    config[:word_tags_to_category] = 
+    align_tags(config[:word_tags], ts)
+    config[:phrase_tags_to_category] =
+    align_tags(config[:phrase_tags], ts)
+  end
+
+  # Align tag configuration.
+  def self.align_tags(tags, tag_sets)
+    wttc = {}
+    tags.each_slice(2) do |desc, tags|
+      category = desc.gsub(',', ' ,').
+      split(' ')[0].downcase
+      tag_sets.each_with_index do |tag_set, i|
+        next unless tags[i]
+        wttc[tags[i]] ||= {}
+        wttc[tags[i]][tag_set] = category
+      end
+    end
+    wttc
+  end
+  
+  def self.hash_to_struct(hash)
+    return hash if hash.keys.
+    select { |k| !k.is_a?(Symbol) }.size > 0
+    struct = Struct.new(
+    *hash.keys).new(*hash.values)
+    hash.each do |key, value|
+      if value.is_a?(Hash)
+        struct[key] =
+        self.hash_to_struct(value)
+      end
+    end
+    struct
+  end
+
+  # Turn on syntactic sugar.
+  def self.sweeten!
+    
+    # Undo this in unsweeten! - # Fix
+    Treat::Entities.module_eval do 
+      self.constants.each do |type|
+        define_singleton_method(type) do |value='', id=nil|
+          const_get(type).build(value, id)
+        end
+      end
+    end
+    
+    return if Treat.core.syntax.sweetened
+    Treat.core.syntax.sweetened = true
+    Treat.core.entities.list.each do |type|
+      next if type == :Symbol
+      kname = cc(type).intern
+      klass = Treat::Entities.const_get(kname)
+      Object.class_eval do
+        define_method(kname) do |val, opts={}|
+          klass.build(val, opts)
+        end
+      end
+    end
+  end
+
+  # Turn off syntactic sugar.
+  def self.unsweeten!
+    return unless Treat.core.syntax.sweetened
+    Treat.core.syntax.sweetened = false
+    Treat.core.entities.list.each do |type|
+      name = cc(type).intern
+      next if type == :Symbol
+      Object.class_eval { remove_method(name) }
+    end
+
+  end
+
+  # Run all configuration.
+  self.configure
+
+end

data/lib/treat/{classification.rb → core/classification.rb}

@@ -1,4 +1,4 @@
-class Treat::Classification
+class Treat::Core::Classification
 
   attr_reader :types
   attr_reader :features

data/lib/treat/{data_set.rb → core/data_set.rb}

@@ -1,7 +1,4 @@
-class Treat::DataSet
-
-  require 'psych'
-  require 'treat/classification'
+class Treat::Core::DataSet
   
   attr_reader :classification
   attr_reader :labels

data/lib/treat/{tree.rb → core/node.rb}

@@ -1,5 +1,5 @@
 # This module provides an abstract tree structure.
-module Treat::Tree
+module Treat::Core
 
   # This class is a node for an N-ary tree data structure
   # with a unique identifier, text value, children, features
@@ -113,7 +113,7 @@ module Treat::Tree
     # node from the children.
     def remove!(ion)
       return nil unless ion
-      if ion.is_a? Treat::Tree::Node
+      if ion.is_a? Treat::Core::Node
         @children.delete(ion)
         @children_hash.delete(ion.id)
         ion.set_as_root!
@@ -203,7 +203,7 @@ module Treat::Tree
     # the supplied dependency type.
     def link(id_or_node, type = nil,
       directed = true, direction = 1)
-      if id_or_node.is_a?(Treat::Tree::Node)
+      if id_or_node.is_a?(Treat::Core::Node)
         id = root.find(id_or_node).id
       else
         id = id_or_node
@@ -220,7 +220,7 @@ module Treat::Tree
 
     # Find the node in the tree with the given id.
     def find(id_or_node)
-      if id_or_node.is_a?(Treat::Tree::Node)
+      if id_or_node.is_a?(Treat::Core::Node)
         id = id_or_node.id
       else
         id = id_or_node
@@ -230,7 +230,7 @@ module Treat::Tree
       end
       self.each do |child|
         r = child.find(id)
-        return r if r.is_a? Treat::Tree::Node
+        return r if r.is_a? Treat::Core::Node
       end
       nil
     end

data/lib/treat/core/server.rb

@@ -0,0 +1,3 @@
+module Treat::Core::Server
+  # To implement.
+end

data/lib/treat/core.rb

@@ -0,0 +1,5 @@
+# Contains the core classes used by Treat.
+module Treat::Core
+  p = Treat.paths.lib + 'treat/core/*.rb'
+  Dir.glob(p).each { |f| require f }
+end

data/lib/treat/entities/abilities/buildable.rb

@@ -3,9 +3,11 @@
 # a string or a numeric object. This class
 # is pretty much self-explanatory.
 module Treat::Entities::Abilities::Buildable
-
+  
+  require 'schiphol'
   require 'fileutils'
-
+  require 'uri'
+  
   # Simple regexps to match common entities.
   WordRegexp = /^[[:alpha:]\-']+$/
   NumberRegexp = /^#?([0-9]+)(\.[0-9]+)?$/
@@ -23,7 +25,9 @@ module Treat::Entities::Abilities::Buildable
   def build(file_or_value, options = {})
 
     fv = file_or_value.to_s
-    if self == Treat::Entities::Document
+    if self == Treat::Entities::Document || 
+      (fv.index('yml') || fv.index('yaml') || 
+      fv.index('xml') || fv.index('mongo'))
       if fv =~ UriRegexp
         from_url(fv, options)
       else
@@ -82,22 +86,18 @@ module Treat::Entities::Abilities::Buildable
       'Cannot create something ' +
       'else than a document from a url.'
     end
-
-    uri = ::URI.parse(url)
-
-    sp = uri.path.split('/')
-    sp.shift if sp[0] == ''
-
-    file = sp[-1]
-    path = sp.size == 1 ?
-    '/' : sp[0..-2].join('/')
-
-    f = Treat::Downloader.download(
-    uri.scheme, uri.host, path, file)
-    options[:default_to] ||= :html
+    
+    f = Schiphol.download(url,
+      :download_folder => Treat.paths.files,
+      :show_progress => Treat.core.verbosity.silence,
+      :rectify_extensions => true,
+      :max_tries => 3
+    )
+    
+    options[:default_to] ||= 'html'
 
     e = from_file(f, options)
-    e.set :url, uri.to_s
+    e.set :url, url.to_s
     e
 
   end
@@ -161,24 +161,17 @@ module Treat::Entities::Abilities::Buildable
 
   # Build a document from a raw or serialized file.
   def from_file(file, options)
-    
-    unless File.readable?(file)
-      raise Treat::Exception,
-      "Path '#{file}' does not "+
-      "point to a readable file."
-    end
 
-    fmt = Treat::Formatters::Readers::Autoselect.
-    detect_format(file, options[:default_to])
-    options[:_format] = fmt
-    
-    if fmt == :yaml || fmt == :yml ||
-      (fmt == :xml && is_treat_xml?(file))
-      f = from_serialized_file(file, options)
+
+    if file.index('yml') || file.index('yaml') || file.index('xml') || file.index('mongo')
+      from_serialized_file(file, options)
     else
-      f = from_raw_file(file, options)
+      fmt = Treat::Workers::Formatters::Readers::Autoselect.
+      detect_format(file, options[:default_to])
+      options[:_format] = fmt
+      from_raw_file(file, options)
     end
-
+    
   end
 
   # Build a document from a raw file.
@@ -190,7 +183,13 @@ module Treat::Entities::Abilities::Buildable
       "Cannot create something else than a " +
       "document from raw file '#{file}'."
     end
-
+    
+    unless File.readable?(file)
+      raise Treat::Exception,
+      "Path '#{file}' does not "+
+      "point to a readable file."
+    end
+    
     d = Treat::Entities::Document.new(file)
 
     d.read(:autoselect, options)
@@ -200,11 +199,29 @@ module Treat::Entities::Abilities::Buildable
   # Build an entity from a serialized file.
   def from_serialized_file(file, options)
 
-    d = Treat::Entities::Document.new(file)
-    d.unserialize(:autoselect, options)
-    d.children[0].set_as_root!
-    d.children[0]
-
+    if file.index('mongo')
+      options[:id] = file.scan(              # Consolidate this
+      /([0-9]+)\.mongo/).first.first
+      from_db(:mongo, options)
+    else
+      unless File.readable?(file)
+        raise Treat::Exception,
+        "Path '#{file}' does not "+
+        "point to a readable file."
+      end
+      d = Treat::Entities::Document.new(file)
+      d.unserialize(:autoselect, options)
+      d.children[0].set_as_root!              # Fix this
+      d.children[0]
+    end
+    
+  end
+  
+  def from_db(adapter, options)
+    id = options[:id]
+    e = self.new(nil, id)
+    e.unserialize(adapter, options)
+    e
   end
 
   # Build any kind of entity from a string.
@@ -217,7 +234,7 @@ module Treat::Entities::Abilities::Buildable
       "collection from a string " +
       "(need a readable file/folder)."
     when :phrase
-      phrase_from_string(string)
+      sentence_or_phrase_from_string(string)
     when :token
       token_from_string(string)
     when :zone
@@ -229,7 +246,7 @@ module Treat::Entities::Abilities::Buildable
         if string.gsub(/[\.\!\?]+/,
           '.').count('.') <= 1 &&
           string.count("\n") == 0
-          phrase_from_string(string)
+          sentence_or_phrase_from_string(string)
         else
           zone_from_string(string)
         end
@@ -245,11 +262,13 @@ module Treat::Entities::Abilities::Buildable
   end
 
   # Build a phrase from a string.
-  def phrase_from_string(string)
+  def sentence_or_phrase_from_string(string)
 
     check_encoding(string)
 
-    if string.count('.!?') >= 1
+    if !(string =~ /[a-zA-Z]+/)
+      Treat::Entities::Fragment.new(string)
+    elsif string.count('.!?') >= 1
       Treat::Entities::Sentence.new(string)
     else
       Treat::Entities::Phrase.new(string)
@@ -300,20 +319,6 @@ module Treat::Entities::Abilities::Buildable
     end
 
   end
-  
-  # Eventually find a better way.
-  def is_treat_xml?(file)
-
-    beginning = nil
-
-    File.open(file) do |w|
-      beginning = w.readlines(200)
-    end
-
-    beginning = beginning.join(' ')
-    beginning.count('<treat>') > 0
-
-  end
 
   def create_collection(fv)
     FileUtils.mkdir(fv)

data/lib/treat/entities/abilities/checkable.rb

@@ -11,8 +11,8 @@ module Treat::Entities::Abilities::Checkable
     return @features[feature] if has?(feature)
     return send(feature) if do_it
     task = caller_method(2) # This is dangerous !
-    g1 = Treat::Categories.lookup(task)
-    g2 = Treat::Categories.lookup(feature)
+    g1 = Treat::Workers.lookup(task)
+    g2 = Treat::Workers.lookup(feature)
 
     raise Treat::Exception,
     "#{g1.type.to_s.capitalize} #{task} " +

data/lib/treat/entities/abilities/comparable.rb

@@ -0,0 +1,21 @@
+module Treat::Entities::Abilities::Comparable
+
+  def compare_with(klass)
+    
+    i = 0; rank_a = nil; rank_b = nil
+    
+    Treat.core.entities.order.each do |type|
+      klass2 = Treat::Entities.const_get(cc(type))
+      rank_a = i if self <= klass2
+      rank_b = i if klass <= klass2
+      next if rank_a && rank_b
+      i += 1
+    end
+    
+    return -1 if rank_a < rank_b
+    return 0 if rank_a == rank_b
+    return 1 if rank_a > rank_b
+    
+  end
+
+end