treat 0.1.1 → 0.1.2

data/lib/treat/lexicalizers/tag/stanford.rb

@@ -3,17 +3,15 @@ module Treat
     module Tag
       class Stanford
         # Require the Ruby-Java bridge.
-        silently do
+        silence_warnings do
           require 'rjb'
-          jar = "#{Treat.bin}/stanford_tagger/stanford-postagger.jar"
-          unless File.readable?(jar)
-            raise "Could not find stanford tagger JAR file in #{jar}."+
-            " You may need to set Treat.bin to a custom value."
+          jar = "#{Treat.bin}/stanford-tagger*/stanford-postagger*.jar"
+          jars = Dir.glob(jar)
+          if jars.empty? || !File.readable?(jars[0])
+            raise "Could not find stanford tagger JAR file (looking in #{jar})."+
+            " You may need to manually download the JAR files and/or set Treat.bin."
           end
-          Rjb::load(
-            "#{Treat.bin}/stanford_tagger/stanford-postagger.jar", 
-            ['-Xms256M', '-Xmx512M']
-          )
+          Rjb::load(jars[0], ['-Xms256M', '-Xmx512M'])
           MaxentTagger = ::Rjb::import('edu.stanford.nlp.tagger.maxent.MaxentTagger')
           Word = ::Rjb::import('edu.stanford.nlp.ling.Word')
           List = ::Rjb::import('java.util.ArrayList')
@@ -43,8 +41,8 @@ module Treat
           else
             model = LanguageToModel[lang]
             if model.nil?
-              raise Treat::Exception "There exists no Stanford" +
-              "tagger model for language #{lang}."
+              raise Treat::Exception, "There exists no Stanford tagger model for " +
+              "the #{Treat::Languages.describe(lang)} language ."
             end
           end
           # Reinitialize the tagger if the options have changed.
@@ -53,15 +51,18 @@ module Treat
             @@taggers[lang] = nil # Reset the tagger
           end
           if @@taggers[lang].nil?
-            model = "#{Treat.bin}/stanford_tagger/models/#{model}"
-            unless File.readable?(model)
-              raise "Could not find a tagger model for language #{lang}: looking in #{model}."
+            model = "#{Treat.bin}/stanford-tagger*/models/#{model}"
+            models = Dir.glob(model)
+            if models.empty? || !File.readable?(models[0])
+              raise "Could not find a tagger model for the " +
+              "#{Treat::Languages.describe(lang)}: looking in #{model}."
             end
             silence_streams(STDOUT, STDERR) do
               @@taggers[lang] =
-              MaxentTagger.new(model)
+              MaxentTagger.new(models[0])
             end
           end
+          entity.set :tag_set, :penn
           list = List.new
           id_list = {}
           i = 0

data/lib/treat/lexicalizers.rb

@@ -7,7 +7,7 @@ module Treat
     module Tag
       extend Group
       self.type = :annotator
-      self.targets = [:phrase, :word]
+      self.targets = [:word]
     end
     module Category
       extend Group

data/lib/treat/object.rb

@@ -0,0 +1,6 @@
+# Make undefining constants publicly available on any object.
+Object.module_eval do
+  def self.const_unset(const)
+    Object.instance_eval { remove_const(const) }
+  end
+end

data/lib/treat/processors/parsers/enju.rb

@@ -55,7 +55,7 @@ module Treat
             text = entity.to_s + '.'
           else
             remove_last = false
-            text = entity.to_s
+            text = entity.to_s.gsub('.', '') + '.' # Fix
           end
           stdin.puts(text + "\n")
           parsed = build(stdout.gets, remove_last)
@@ -120,7 +120,7 @@ module Treat
                       new_attributes[:enju_cat] = value
                       xcat = attributes['xcat'].split(' ')[0]
                       xcat ||= ''
-                      tags = Treat::Resources::Tags::EnjuCatXcatToPTB.select do |m|
+                      tags = Treat::Languages::English::EnjuCatXcatToPTB.select do |m|
                         m[0] == value && m[1] == xcat
                       end
                       if tags.empty?
@@ -144,6 +144,7 @@ module Treat
             # Handle naming conventions.
             if attributes.has_key?('pos')
               new_attributes[:tag] = new_attributes[:pos]
+              new_attributes[:tag_set] = :penn
               new_attributes.delete :pos
             end
             # Create the appropriate entity for the

data/lib/treat/processors/parsers/stanford.rb

@@ -3,22 +3,24 @@ module Treat
     module Parsers
       class Stanford
         # Require the Ruby-Java bridge.
-        silently { require 'rjb' }
-        jar = "#{Treat.bin}/stanford_parser/stanford-parser.jar"
-        unless File.readable?(jar)
-          raise "Could not find stanford parser JAR file in #{jar}."+
-          " You may need to set Treat.bin to a custom value."
+        silence_warnings { require 'rjb' }
+        jar = "#{Treat.bin}/stanford-parser*/stanford-parser*.jar"
+        jars = Dir.glob(jar)
+        if jars.empty? || !File.readable?(jars[0])
+          raise "Could not find stanford parser JAR file (looking in #{jar})"+
+          " You may need to manually download the JAR files and/or set Treat.bin."
         end
-        Rjb::load(jar, ['-Xms256M', '-Xmx512M'])
+        Rjb::load(jars[0], ['-Xms256M', '-Xmx512M'])
         LexicalizedParser = ::Rjb::import('edu.stanford.nlp.parser.lexparser.LexicalizedParser')
         @@parsers = {}
         def self.parse(entity, options = {})
-          lang = Treat::Resources::Languages.describe(entity.language).to_s
-          pcfg = "#{Treat.bin}/stanford_parser/grammar/#{lang.upcase}PCFG.ser.gz"
-          unless File.readable?(pcfg)
-            raise "Could not find a language model for #{lang}: looking in #{pcfg}."
+          lang = Treat::Languages.describe(entity.language).to_s.upcase
+          pcfg = "#{Treat.bin}/stanford-parser*/grammar/#{lang.upcase}PCFG.ser.gz"
+          pcfgs = Dir.glob(pcfg)
+          if pcfgs.empty? || !File.readable?(pcfgs[0])
+            raise "Could not find a language model for #{lang.downcase} (looking in #{pcfg})."
           end
-          @@parsers[lang] ||= LexicalizedParser.new(pcfg) # Fix - check that exists.
+          @@parsers[lang] ||= LexicalizedParser.new(pcfgs[0])
           parse = @@parsers[lang].apply(entity.to_s)
           entity.remove_all!          
           recurse(parse, entity)
@@ -41,12 +43,13 @@ module Treat
               return recurse(java_node.children[0], ruby_node)
             end
             java_node.children.each do |java_child|
-              dependencies = java_child.dependencies.iterator
+              # dependencies = java_child.dependencies.iterator
               # while dependencies.has_next
                 #dependency = dependencies.next
               # end
               ruby_child = Treat::Entities::Phrase.new
               ruby_child.set :tag, java_child.value 
+              ruby_child.set :tag_set, :penn
               ruby_node << ruby_child
               unless java_child.children.empty?
                 recurse(java_child, ruby_child)

data/lib/treat/processors/segmenters/punkt.rb

@@ -9,7 +9,7 @@ module Treat
       # Unsupervised Multilingual Sentence Boundary Detection. 
       # Computational Linguistics 32: 485-525.
       class Punkt
-        silently { require 'punkt-segmenter' }
+        silence_warnings { require 'punkt-segmenter' }
         # Hold one copy of the segmenter per language.
         @@segmenters = {}
         # Hold only one trainer per language.

data/lib/treat/processors/segmenters/stanford.rb

@@ -3,13 +3,15 @@ module Treat
     module Segmenters
       class Stanford
         # Require the Ruby-Java bridge.
-        silently do
+        silence_warnings do
           require 'rjb'
-          jar = "#{Treat.bin}/stanford_parser/stanford-parser.jar"
-          unless File.readable?(jar)
-            raise "Could not find stanford parser JAR file in #{jar}."+
-            " You may need to set Treat.bin to a custom value."
+          jar = "#{Treat.bin}/stanford-parser*/stanford-parser*.jar"
+          jars = Dir.glob(jar)
+          if jars.empty? || !File.readable?(jars[0])
+            raise "Could not find stanford parser JAR file (lookin in #{jar})."+
+            " You may need to manually download the JAR files and/or set Treat.bin."
           end
+          ::Rjb::load(jars[0])
           DocumentPreprocessor =
           ::Rjb::import('edu.stanford.nlp.process.DocumentPreprocessor')
           StringReader = ::Rjb::import('java.io.StringReader')

data/lib/treat/processors/segmenters/tactful.rb

@@ -11,7 +11,7 @@ module Treat
       # Project website:
       class Tactful
         # Require the 'tactful_tokenizer' gem.
-        silently { require 'tactful_tokenizer' }
+        silence_warnings { require 'tactful_tokenizer' }
         # Somewhere in the depths of the code this is defined...
         String.class_eval { undef :tokenize }
         # Keep only one copy of the segmenter.

data/lib/treat/processors/tokenizers/multilingual.rb

@@ -8,13 +8,13 @@ module Treat
         # Hold one tokenizer per language.
         @@tokenizers = {}
         # Require the 'tokenizer' gem.
-        silently { require 'tokenizer' }
+        silence_warnings { require 'tokenizer' }
         # Perform the tokenization of English, German or French text.
         # Options:
         # :language => (Symbol) Force a language for the tokenizer.
         def self.tokenize(entity, options = {})
           lang = options[:language] ? options[:language] : entity.language
-          lang = Treat::Resources::Languages.find(lang, 1)    
+          lang = Treat::Languages.find(lang, 1)    
           if @@tokenizers[lang].nil?
             @@tokenizers[lang] = ::Tokenizer::Tokenizer.new(lang)
           end

data/lib/treat/processors/tokenizers/stanford.rb

@@ -3,14 +3,16 @@ module Treat
     module Tokenizers
       class Stanford
         # Require the Ruby-Java bridge.
-        silently do
+        silence_warnings do
           require 'rjb'
           # Load the Stanford Parser Java files.
-          jar = "#{Treat.bin}/stanford_parser/stanford-parser.jar"
-          unless File.readable?(jar)
-            raise "Could not find stanford parser JAR file in #{jar}."+
-            " You may need to set Treat.bin to a custom value."
+          jar = "#{Treat.bin}/stanford-parser/stanford-parser.jar"
+          jars = Dir.glob(jar)
+          if jars.empty? || !File.readable?(jars[0])
+            raise "Could not find stanford parser JAR file (looking in #{jar})."+
+            " You may need to manually download the JAR files and/or set Treat.bin."
           end
+          ::Rjb::load(jars[0])
           # Load the Stanford Parser classes.
           PTBTokenizer = ::Rjb::import('edu.stanford.nlp.process.PTBTokenizer')
           CoreLabelTokenFactory = ::Rjb::import('edu.stanford.nlp.process.CoreLabelTokenFactory')

data/lib/treat/visitable.rb

@@ -20,7 +20,8 @@ module Treat
           return klass.send(method, self, options)
         end
       else
-        raise "This type of visitor cannot visit a #{self.class}."
+        raise NAT::Exception,
+        "This type of visitor cannot visit a #{self.class}."
       end
     end
   end

data/lib/treat.rb

@@ -1,58 +1,93 @@
-# This file requires all source code files for the Treat module.
-
 #
-# Main Treat namespace.
+# Main namespace for Treat modules.
 #
-# Textual model:
+# 1. Entities
 #
-# - Tree - Contains abstract tree node and leaf structures.
-# - Entities - Contains concrtypee node and leaf structures
-#   that represent textual entities.
+#   Entities are Tree structures that represent any textual 
+#   entity (from a collection of texts down to an individual
+#   word) with a value, features, children and edges linking
+#   it to other textual entities. Sugar provides syntactic sugar
+#   for Entities and can be enabled by running Treat.edulcorate.
+# 
+#   Here are some example of how to create entities:
 #
-# Algorithm namespaces:
+#   c = Collection 'folder_with_documents'
+#   d = Document 'filename.txt' # (or PDF, html, xml, png, jpg, gif).
+#   p = Paragraph 'A short story. The end.'
+#   s = Sentence 'That is not a sentence.'
+#   w = Word 'fox'
+#  
+#   Here's a full list of entities (subtypes in parentheses): 
+#   Collection, Document, Zone (Section, Title, Paragraph or List),
+#   Sentence, Constituent (Phrase or Clause), Token (Word, Number,
+#   Symbol or Punctuation).
+# 
+# 2. Proxies
+# 
+#   Proxies allow the Treat functions to be called on the core
+#   Ruby classes String, Numeric and Array. They build the entity 
+#   corresponding to the supplied raw text and send the requested 
+#   function to it.
+#   
+#   For example,
 #
-# - Dtypeectors - Namespace for language, encoding, and format
-#   detectors.
-# - Extractors - Namespace for algorithms that extract
-#   information from entities.
-# - Formatters - Namespace for algorithms that handle
-#   conversion to and from different formats.
-# - Inflectors - Namespace for algorithms that supply
-#   the base form, inflections and declensions of a word.
-# - Lexicalizers - Namespace for algorithms that supply
-#   lexical information about a word (part of speech,
-#   synstypes, klass.)
-# - Processors - Namespace for algorithms that process an
-#   entity into a tree of sub-entities.
+#       'fox'.tag
+#
+#   Is equivalent to:
+#
+#       w = Word 'fox'
+#       w.tag
+# 
+# 3. Functions
 #
-# Other modules:
+#   A class is defined for each implemented algorithm performing a given
+#   task. These classes are clustered into groups of algorithms performing
+#   the same given task (Group), and the groups are clustered into Categories 
+#   of groups performing related tasks.
 #
-# - Group - Creates functions for algorithm groups.
-# - Proxies - Provide proxies for Treat functions on String,
-#   Numeric and Array classes.
-# - Utilities - Supply utility functions used across the library.
+#   Here are the different Categories:
+# 
+# - Detectors - Category for language, encoding, and format 
+#   detectors.
+# - Extractors - Category for algorithms that extract information 
+#   from entities.
+# - Formatters - Category for algorithms that handle conversion 
+#   to and from different formats.
+# - Inflectors - Category for algorithms that supply the base 
+#   form, inflections and declensions of a word.
+# - Lexicalizers - Category for algorithms that supply lexical 
+#   information about a word (part of speech, synsets, word categories).
+# - Processors - Namespace for algorithms that process collections and 
+#   documents into trees.
+#
+# 3. Linguistic resources
+# 
+#   The Languages module contains linguistic information about 
+#   languages (full ISO-639-1 and 2 language list, tag alignments 
+#   for three treebanks, word categories, etc.)
+#  
+# 4. Mixins for entities.
+# 
+#  Buildable, Delegatable, Visitable and Registrable are
+#  or extended by Entity and provide it with the ability to be built, 
+#  to delegate function calls, to accept visitors and to maintain a 
+#  token registry, respectively.
 #
+# 5. Exception
+#  
+#  Exception defines a custom exception for the Treat module.
+# 
 module Treat
 
   # Make sure that we are running on Ruby 1.9 or higher.
   if RUBY_VERSION <= '1.9'
     raise 'Treat requires Ruby 1.9 or higher.'
   end
-
+  
   # The current version of Treat.
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 
-  # Require all files for the Treat library.
-  require 'treat/exception'
-  require 'treat/utilities'
-  require 'treat/resources'
-  require 'treat/entities'
-  require 'treat/categories'
-  require 'treat/proxies'
-
-  # Provides syntactic sugar.
-  require 'treat/sugar'
-  extend Sugar
+#  $LOAD_PATH << '/ruby/treat/lib/' # Remove for release
   
   # Create class variables for the Treat module.
   class << self
@@ -67,25 +102,41 @@ module Treat
     attr_accessor :language_detection_level
     # String - main folder for executable files.
     attr_accessor :bin
+    # String - folder of this file.
+    attr_accessor :lib
+    # String - folder for tests.
+    attr_accessor :test
+    # String - folder for temp files.
+    attr_accessor :tmp
   end
-
-  # Folder paths.
-  @@lib = File.dirname(__FILE__)
-  @@test = @@lib + '/../test/'
-  @@tmp = @@lib + '/../tmp/'
-  @@bin = @@lib + '/../bin'
-  def self.lib; @@lib; end
-  def self.test; @@test; end
-  def self.tmp; @@tmp; end
-
-  # Stype the default language to english.
+  
+  # Set the default language to english.
   self.default_language = :eng
-  # Stype the default encoding to utf-8.
+  # Set the default encoding to utf-8.
   self.default_encoding = :utf_8
   # Turn language detection off by default.
   self.detect_language = false
-  # Dtypeect the language once per text by default.
+  # Detect the language once per text by default.
   self.language_detection_level = :text
-  # Stype the bin path to the gem's bin folder by default.
-  self.bin = @@bin  
+  # Set the lib path to that of this file.
+  self.lib = File.dirname(__FILE__)
+  # Set the paths to the bin, test and tmp folders.
+  self.bin = self.lib + '/../bin/'
+  self.test = self.lib + '/../test/'
+  self.tmp = self.lib + '/../tmp/'
+  
+  # Require modified core classes.
+  require 'treat/object'
+  require 'treat/kernel'
+  
+  # Require all files for the Treat library.
+  require 'treat/exception'
+  require 'treat/languages'
+  require 'treat/entities'
+  require 'treat/categories'
+  require 'treat/proxies'
+  require 'treat/sugar'
+  
+  extend Sugar
+
 end

data/test/tc_entity.rb

@@ -16,18 +16,23 @@ module Treat
         @det = Treat::Entities::Word.new('The')
         @det.set :cat, :determiner
         @det.set :tag, 'DT'
+        @det.set :tag_set, :penn
         @adj = Treat::Entities::Word.new('lazy')
         @adj.set :cat, :adjective
         @adj.set :tag, 'JJ'
+        @adj.set :tag_set, :penn
         @noun = Treat::Entities::Word.new('fox')
         @noun.set :cat, :noun
         @noun.set :tag, 'NN'
+        @noun.set :tag_set, :penn
         @aux = Treat::Entities::Word.new('is')
         @aux.set :cat, :verb
         @aux.set :tag, 'VBZ'
+        @aux.set :tag_set, :penn
         @verb = Treat::Entities::Word.new('running')
         @verb.set :cat, :verb
         @verb.set :tag, 'VBG'
+        @verb.set :tag_set, :penn
         @dot = Treat::Entities::Punctuation.new('.')
         
         @text << @sentence << [@noun_phrase, @verb_phrase, @dot]

data/test/tc_resources.rb

@@ -1,12 +1,12 @@
 module Treat
   module Tests
-    class TestResources < Test::Unit::TestCase
+    class TestLanguages < Test::Unit::TestCase
 
       def test_languages
-        assert_equal :eng, Treat::Resources::Languages.find(:english, 2)
-        assert_equal :en, Treat::Resources::Languages.find(:english, 1)
-        assert_equal :english, Treat::Resources::Languages.describe(:eng)
-        assert_equal :english, Treat::Resources::Languages.describe(:en)
+        assert_equal :eng, Treat::Languages.find(:english, 2)
+        assert_equal :en, Treat::Languages.find(:english, 1)
+        assert_equal :english, Treat::Languages.describe(:eng)
+        assert_equal :english, Treat::Languages.describe(:en)
       end
       
       def test_tags

data/test/tc_treat.rb

@@ -29,8 +29,7 @@ module Treat
       
       def test_modules_loaded?
         ['exception',
-        'utilities',
-        'resources',
+        'languages',
         'entities',
         'feature',
         'category',

data/test/tests.rb

@@ -1,11 +1,12 @@
 require 'test/unit'
 
-# $LOAD_PATH << '/ruby/treat/test/' # Remove for production
 $:.unshift File.join(File.dirname(__FILE__), "..", "lib")
 
 require 'treat'
 require 'texts'
 
+# Treat.bin = '/ruby/nat/bin' # Remove for release
+
 require 'tc_treat'
 require 'tc_tree'
 require 'tc_entity'