treat 0.1.1

data/lib/treat/utilities.rb

@@ -0,0 +1,127 @@
+module Treat
+  # Provides utility functions used across the library.
+  module Utilities
+    # Require file utilities.
+    require 'fileutils'
+    # Returns the platform we are running on.
+    def self.platform
+      RUBY_PLATFORM.split("-")[1]
+    end
+    # Runs a block of code silently, i.e. without
+    # expressing warnings even in verbose mode.
+    # Rename to silence_streamsings.
+    def self.silently(&block)
+      warn_level = $VERBOSE
+      $VERBOSE = nil
+      result = block.call
+      $VERBOSE = warn_level
+      result
+    end
+    def self.silence_streams(*streams)
+      yield
+    end
+    # Create a temporary file which is deleted
+    # after execution of the block.
+    require 'tempfile'
+    def self.create_temp_file(ext, value = nil, &block)
+      tmp = Tempfile.new(['', ".#{ext.to_s}"], Treat.tmp)
+      tmp.puts(value) if value
+      block.call(tmp.path)
+    end
+    # A list of acronyms used in class names within
+    # the program. These do not CamelCase; they
+    # CAMELCASE.
+    @@acronyms = ['XML', 'HTML', 'YAML', 'UEA', 'LDA', 'PDF', 'GOCR', 'Treat'].join('|')
+    @@cc_cache = {}
+    # Convert un_camel_case to CamelCase.
+    def self.camel_case(o_phrase)
+      phrase = o_phrase.to_s.dup
+      return @@cc_cache[o_phrase] if @@cc_cache[o_phrase]
+      phrase.gsub!(/#{@@acronyms.downcase}[^a-z]+/) { |a| a.upcase }
+      phrase.gsub!(/^[a-z]|_[a-z]/) { |a| a.upcase }
+      phrase.gsub!('_', '')
+      @@cc_cache[o_phrase] = phrase
+      phrase
+    end
+    @@ucc_cache = {}
+    # Convert CamelCase to un_camel_case.
+    def self.un_camel_case(o_phrase)
+      phrase = o_phrase.to_s.dup
+      return @@ucc_cache[o_phrase] if @@ucc_cache[o_phrase]
+      phrase.gsub!(/#{@@acronyms}/) { |a| a.downcase.capitalize }
+      phrase.gsub!(/[A-Z]/) { |p| '_' + p.downcase  }
+      phrase = phrase[1..-1] if phrase[0] == '_'
+      @@ucc_cache[o_phrase] = phrase
+      phrase
+    end
+    # Return the levensthein distance between two stringsm
+    # taking into account the costs of insertion, deletion,
+    # and substitution. Stolen from:
+    # http://ruby-snippets.heroku.com/string/levenshtein-distance
+    def self.levenshtein(first, other, ins=1, del=1, sub=1)
+      return nil if first.nil? || other.nil?
+      dm = []
+      dm[0] = (0..first.length).collect { |i| i * ins}
+      fill = [0] * (first.length - 1)
+      for i in 1..other.length
+        dm[i] = [i * del, fill.flatten]
+      end
+      for i in 1..other.length
+        for j in 1..first.length
+          dm[i][j] = [
+            dm[i-1][j-1] + (first[i-1] == other[i-1] ? 0 : sub),
+            dm[i][j-1] + ins,
+            dm[i-1][j] + del
+          ].min
+        end
+      end
+      dm[other.length][first.length]
+    end
+    # Search the list to see if there are words
+    # similar to name. If yes, return a string
+    # saying "Did you mean ... ?"
+    def self.did_you_mean?(list, name)
+      msg = ''
+      sugg = []
+      list.each do |element|
+        l = levenshtein(element,name)
+        if  l > 0 && l < 2
+          sugg << element
+        end
+      end
+      unless sugg.empty?
+        if sugg.size == 1
+          msg += " Perhaps you meant '#{sugg[0]}' ?"
+        else
+          sugg_quote = sugg[0..-2].map {|x| '\'' + x + '\''}
+          msg += " Perhaps you meant #{sugg_quote.join(', ')}," +
+          " or '#{sugg[-1]}' ?"
+        end
+      end
+      msg
+    end
+    def self.caller_method(n = 3)
+      at = caller(n).first
+      /^(.+?):(\d+)(?::in `(.*)')?/ =~ at
+      :"#{Regexp.last_match[3]}"
+    end
+  end
+end
+
+# Make undefining constants publicly
+# available on any object.
+Object.module_eval do
+  def self.const_unset(const); Object.instance_eval { remove_const(const) }; puts const; end
+end
+
+# Make the most common utility functions available in the global scope.
+def create_temp_file(ext, value = nil, &block)
+  Treat::Utilities.create_temp_file(ext, value) { |f| block.call(f) }
+end
+def silence_streams(*streams); Treat::Utilities.silence_streams(*streams) { yield }; end
+def silently(&block); Treat::Utilities.silently { block.call }; end
+def cc(w); Treat::Utilities.camel_case(w); end
+def ucc(w); Treat::Utilities.un_camel_case(w); end
+def cl(n); n.to_s.split('::')[-1]; end
+def did_you_mean?(l, e); Treat::Utilities.did_you_mean?(l, e); end
+def caller_method(n = 3); Treat::Utilities.caller_method(n); end

data/lib/treat/visitable.rb

@@ -0,0 +1,27 @@
+module Treat
+  # Make a tree visitable by implementing the method #accept.
+  module Visitable
+    # Accept a visitor implemented by klass, which is
+    # found in the supplied group, and call method on it.
+    def accept(group, klass, method, options)
+      if group.has_target?(self.class)
+        if group.type == :transformer
+          if has_children?
+            @children.each do |entity|
+              if group.has_target?(entity.class)
+                entity.accept(group, klass, method, options)
+              end
+            end
+          else
+            klass.send(method, self, options)
+          end
+          return self
+        else
+          return klass.send(method, self, options)
+        end
+      else
+        raise "This type of visitor cannot visit a #{self.class}."
+      end
+    end
+  end
+end

data/test/profile.rb

@@ -0,0 +1,2 @@
+require 'unprof'
+require 'tests'

data/test/tc_detectors.rb

@@ -0,0 +1,27 @@
+module Treat
+  module Tests
+    class TestDetectors < Test::Unit::TestCase
+      
+      def setup
+        @doc = Treat::Tests::EnglishLongDoc
+      end
+      
+      def test_format_detectors
+        assert_equal :txt, @doc.format
+      end
+      
+      def test_encoding_detectors
+        assert_equal :utf_8, @doc.encoding(:native)
+        assert_equal :utf_8, @doc.encoding(:r_chardet19)
+      end
+      
+      def test_language_detectors
+        assert_equal Treat.default_language, @doc.language
+        Treat.detect_language = true
+        assert_equal :eng, @doc.language
+        Treat.detect_language = false
+      end
+    end
+
+  end
+end

data/test/tc_entity.rb

@@ -0,0 +1,105 @@
+module Treat
+  module Tests
+    class TestEntity < Test::Unit::TestCase
+      def setup
+        @text = Treat::Entities::Text.new
+        
+        @sentence = Treat::Entities::Sentence.new
+        
+        @noun_phrase = Treat::Entities::Phrase.new
+        @noun_phrase.set :tag, 'NP'
+        @verb_phrase = Treat::Entities::Phrase.new
+        @verb_phrase.set :tag, 'VP'
+        @adj_phrase = Treat::Entities::Phrase.new
+        @adj_phrase.set :tag, 'ADJP'
+        
+        @det = Treat::Entities::Word.new('The')
+        @det.set :cat, :determiner
+        @det.set :tag, 'DT'
+        @adj = Treat::Entities::Word.new('lazy')
+        @adj.set :cat, :adjective
+        @adj.set :tag, 'JJ'
+        @noun = Treat::Entities::Word.new('fox')
+        @noun.set :cat, :noun
+        @noun.set :tag, 'NN'
+        @aux = Treat::Entities::Word.new('is')
+        @aux.set :cat, :verb
+        @aux.set :tag, 'VBZ'
+        @verb = Treat::Entities::Word.new('running')
+        @verb.set :cat, :verb
+        @verb.set :tag, 'VBG'
+        @dot = Treat::Entities::Punctuation.new('.')
+        
+        @text << @sentence << [@noun_phrase, @verb_phrase, @dot]
+        @noun_phrase << [@det, @adj_phrase, @noun]
+        @adj_phrase << @adj
+        @verb_phrase << [@aux, @verb]
+      end
+
+      def test_respond_to_missing
+      
+      end
+      
+      def test_registrable
+        assert_equal @text.token_registry, @verb.token_registry
+        assert_equal @noun, @text.token_registry[:id][@noun.id]
+        assert_equal [@noun], @text.token_registry[:value][@noun.value]  
+      end
+
+
+      def test_delegatable_visitable
+        assert_raise(Treat::Exception) do 
+          @text.encoding(:nonexistent)
+        end
+        assert_nothing_raised do
+          @text.format
+        end
+      end
+      
+      def test_type
+        assert_equal :text, @text.type
+      end
+      
+      def test_printers
+        assert_nothing_raised do
+          @text.to_s
+          @text.to_string
+          @text.short_value
+          @text.inspect
+        end
+      end
+
+      def test_magic_methods
+        assert_equal @sentence, @text.sentence
+        assert_equal [@sentence], @text.sentences
+        assert_equal 1, @text.sentence_count
+        
+        assert_equal [@det], @text.words_with_value('The')
+        assert_equal [@verb], @text.words_with_tag('VBG')
+        
+        assert_equal @noun, @text.noun
+        assert_equal [@aux, @verb], @text.verbs
+        assert_equal 6, @text.token_count
+        
+        @text.each_sentence do |s|
+          assert_equal @sentence, s
+        end
+        @text.each_noun do |n|
+          assert_equal @noun, n
+        end
+        @text.each_with_value('The') do |x|
+          assert_equal @det, x
+        end
+        
+        assert_equal @sentence, @noun.parent_sentence
+      end
+
+      def test_features
+        @verb.set :test, :test
+        assert_equal :test,  @verb.test
+        assert_raise(Treat::Exception) { @verb.nonexistent } 
+      end
+      
+    end
+  end
+end

data/test/tc_extractors.rb

@@ -0,0 +1,48 @@
+module Treat
+  module Tests
+    class TestExtractors < Test::Unit::TestCase
+
+      def setup
+        @time = Treat::Tests::EnglishTime
+        @date = Treat::Tests::EnglishDate
+        @doc = Treat::Tests::EnglishLongDoc
+        @word = Treat::Tests::EnglishWord
+      end
+
+      def test_time
+        assert_nothing_raised { @date.time(:chronic) }
+        assert_nothing_raised { @date.time(:native) }
+        assert_nothing_raised { @date.time(:nickel) }
+      end
+
+      def test_topic_words
+        assert_nothing_raised { @doc.topic_words(:lda) }
+      end
+
+
+      def test_named_entity
+        # assert_nothing_raised { @doc.named_entity(:stanford) }
+        # assert_nothing_raised { @doc.named_entity(:abner) }
+      end
+
+      def test_key_sentences
+        topics = @doc.topic_words(:lda)
+        assert_nothing_raised { @doc.key_sentences(:topics_frequency, topics) }
+      end
+
+      def test_topics
+        assert_nothing_raised { @doc.topics(:reuters) }
+      end
+
+      def test_statistics
+        @doc.chunk.segment(:tactful).tokenize
+
+        assert_nothing_raised { @doc.statistics(:frequency_of, value: 'the') }
+        assert_nothing_raised { @word.statistics(:frequency) }
+        # assert_nothing_raised { @doc.statistics(:position_in) }
+        # assert_nothing_raised { @doc.statistics(:transition_matrix) }
+        # assert_nothing_raised { @doc.statistics(:transition_probability) }
+      end
+    end
+  end
+end

data/test/tc_formatters.rb

@@ -0,0 +1,46 @@
+module Treat
+  module Tests
+    class TestFormatters < Test::Unit::TestCase
+
+      def setup
+        @doc = Treat::Tests::EnglishShortDoc
+        @html_doc = Treat::Tests::EnglishHtmlDoc
+        @sentence = Treat::Tests::EnglishSentence
+      end
+
+      def test_readers
+        # How should we test this?
+      end
+
+
+      def test_serializers_and_unserializers
+        create_temp_file('yml') do |tmp|
+          @doc.serialize(:yaml).save(tmp)
+          doc = Treat::Entities::Document(tmp)
+          assert_equal File.read(tmp).length, 
+          doc.serialize(:yaml).length
+        end
+        create_temp_file('xml') do |tmp|
+          @doc.serialize(:xml).save(tmp)
+          doc = Treat::Entities::Document(tmp)
+          assert_equal File.read(tmp).length, 
+          doc.serialize(:xml).length
+        end
+      end
+
+      def test_visualizers
+        assert_nothing_raised { @doc.visualize(:tree) }
+        # assert_nothing_raised { @doc.visualize(:html) }
+        assert_nothing_raised { @doc.visualize(:dot) }
+        assert_nothing_raised { @doc.visualize(:inspect) }
+        assert_nothing_raised { @doc.visualize(:short_value) }
+        assert_nothing_raised { @sentence.visualize(:standoff) }
+      end
+
+      def test_cleaners
+        assert_nothing_raised { @html_doc.clean(:html) }
+      end
+      
+    end
+  end
+end

data/test/tc_inflectors.rb

@@ -0,0 +1,39 @@
+module Treat
+  module Tests
+    class TestInflectors < Test::Unit::TestCase
+
+      def setup
+        @word = Treat::Tests::EnglishWord
+        @number = Treat::Tests::Number
+        @verb = Treat::Tests::EnglishVerb
+        @noun = Treat::Tests::EnglishNoun
+      end
+
+      def test_lemmatizers
+        # Not implemented yet.
+      end
+
+      def test_stemmers
+        assert_equal 'run', @word.stem(:porter)
+        assert_equal 'run', @word.stem(:porter_c)
+        assert_equal 'run', @word.stem(:uea)
+      end
+    end
+
+    def test_conjugators
+      assert_equal 'running', @verb.present_participle
+      assert_equal 'run', @verb.infinitive
+      assert_equal 'run', @verb.plural
+    end
+
+    def test_declensors
+      assert_equal 'geese', @noun.plural
+    end
+
+    def test_ordinal_and_cardinal_words
+      assert_equal 'twenty', @number.cardinal_words
+      assert_equal 'twentieth', @number.ordinal_words
+    end
+
+  end
+end

data/test/tc_lexicalizers.rb

@@ -0,0 +1,39 @@
+module Treat
+  module Tests
+    class TestLexicalizers < Test::Unit::TestCase
+      
+      def setup
+        @word = Treat::Tests::EnglishWord
+        @sentence = Treat::Tests::EnglishSentence.parse
+      end
+      
+      def test_category
+        assert_equal :verb, @word.category(:from_tag)
+      end
+      
+      def test_synsets
+        # assert_nothing_raised { @word.synsets(:rita_wn) }
+        assert_nothing_raised { @word.synsets(:wordnet) }
+        assert_nothing_raised { @word.synonyms(:wordnet) }
+        assert_nothing_raised { @word.antonyms(:wordnet) }
+        assert_nothing_raised { @word.hyponyms(:wordnet) }
+        assert_nothing_raised { @word.hypernyms(:wordnet) }
+      end
+      
+      def test_linkages
+        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :main_verb) }
+        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :subject) }
+        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :object) }
+        assert_nothing_raised { @sentence.linkages(:naive, :linkage => :patient) }
+      end
+      
+      def test_taggers
+        assert_nothing_raised { @word.tag(:brill) }
+        assert_nothing_raised { @word.tag(:lingua) }
+        assert_nothing_raised { @word.tag(:stanford) }
+      end
+      
+    end
+
+  end
+end