treat 1.2.0 → 2.0.0rc1

data/lib/treat/workers/processors/tokenizers/stanford.rb

@@ -1,41 +1,57 @@
-# A wrapper for the Stanford parser's
-# Penn-Treebank style tokenizer.
+# Tokenization provided by Stanford Penn-Treebank style 
+# tokenizer. Most punctuation is split from adjoining words,
+# verb contractions and the Anglo-Saxon genitive of nouns are 
+# split into their component morphemes, and each morpheme is 
+# tagged separately. N.B. Contrary to the standard PTB 
+# tokenization, double quotes (") are NOT changed to doubled 
+# single forward- and backward- quotes (`` and '') by default.
 class Treat::Workers::Processors::Tokenizers::Stanford
 
-  require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load
   
+  # Default options for the tokenizer.
+  DefaultOptions = {
+    directional_quotes: false,
+    escape_characters: false
+  }
+  
+  # Hold one instance of the tokenizer.
   @@tokenizer = nil
 
-  # Tokenize the entity using a Penn-Treebank
-  # style tokenizer.
+  # Perform tokenization of the entity and add
+  # the resulting tokens as its children.
   #
-  # Options: none.
+  # Options:
+  # - (Boolean) :directional_quotes => Whether
+  # to attempt to get correct directional quotes,
+  # replacing "..." by ``...''. Off by default.
   def self.tokenize(entity, options = {})
-
-    entity.check_hasnt_children
-
-    s = entity.to_s
-    
+    options = DefaultOptions.merge(options)
     @@tokenizer ||=
     ::StanfordCoreNLP.load(:tokenize)
-    text =
-    ::StanfordCoreNLP::Text.new(s)
+    entity.check_hasnt_children
+    text = ::StanfordCoreNLP::
+    Text.new(entity.to_s)
     @@tokenizer.annotate(text)
-
-    add_tokens(entity, text.get(:tokens))
-
+    add_tokens(entity, text.get(:tokens), options)
   end
 
   # Add the tokens to the entity.
-  def self.add_tokens(entity, tokens)
+  def self.add_tokens(entity, tokens, options)
     tokens.each do |token|
       val = token.value
-      val = '(' if val == '-LRB-'     # Fix for other special chars
-      val = ')' if val == '-RRB'
-      t = Treat::Entities::Token.
-      from_string(token.value)
-      entity << t
+       unless options[:escape_characters]
+         Treat.tags.ptb.escape_characters.
+            each do |key, value|
+             val.gsub!(value, key)
+           end
+       end
+      unless options[:directional_quotes]
+        val.gsub!(/``/,'"') 
+        val.gsub!(/''/,'"')
+      end
+      entity << Treat::Entities::Token.
+      from_string(val)
     end
   end
 

data/lib/treat/workers/retrievers/indexers/ferret.rb

@@ -17,9 +17,13 @@ class Treat::Workers::Retrievers::Indexers::Ferret
   # Annotates the collection with the path to the
   # index for future use (e.g. in searching).
   def self.index(collection, options = {})
+
+    unless collection.get(:folder)
+      raise Treat::Exception,
+      "Only collections stored on disk " +
+      "can currently be indexed with Ferret."
+    end
     
-    # FIXME - what if the collection is stored
-    # in a database?
     path = "#{collection.folder}/.index"
     return path if FileTest.directory?(path)
     
@@ -45,7 +49,6 @@ class Treat::Workers::Retrievers::Indexers::Ferret
     end
     
     path
-    
   end
   
 end

data/lib/treat/workers/retrievers/searchers/ferret.rb

@@ -10,7 +10,7 @@ class Treat::Workers::Retrievers::Searchers::Ferret
   require 'find'
   
   DefaultOptions = {
-    :q => nil,
+    :query => nil,
     :limit => :all,
     :callback => nil
   }
@@ -30,7 +30,7 @@ class Treat::Workers::Retrievers::Searchers::Ferret
       "This collection must be indexed to be searchable."
     end
     
-    unless options[:q]
+    unless options[:query]
       raise Treat::Exception,
       'You must set a query by using the :q option.'
     end

data/lib/treat/workers/workers.rb

@@ -0,0 +1,6 @@
+module Treat::Workers
+  # Require mixins for workers.
+  require_relative 'categorizable'
+  # Make all workers categorizable.
+  extend Treat::Workers::Categorizable
+end

data/lib/treat.rb

@@ -1,36 +1,22 @@
+# Treat is a toolkit for natural language 
+# processing and computational linguistics 
+# in Ruby. The Treat project aims to build 
+# a language- and algorithm- agnostic NLP 
+# framework for Ruby with support for tasks 
+# such as document retrieval, text chunking, 
+# segmentation and tokenization, natural 
+# language parsing, part-of-speech tagging, 
+# keyword mining and named entity recognition.
+#
+# Author: Louis-Antoine Mullie (c) 2010-12.
+# 
+# Released under the General Public License.
 module Treat
   
-  # Treat requires Ruby >= 1.9.2
-  if RUBY_VERSION < '1.9.2'
-    raise "Treat requires Ruby version 1.9.2 " +
-    "or higher, but current is #{RUBY_VERSION}."
-  end
-  
-  # Custom exception class.
-  class Exception < ::Exception; end
-  
-  # Load configuration options.
-  require 'treat/config'
-  # Load all workers.
-  require 'treat/helpers'
-  # Require library loaders.
-  require 'treat/loaders'
-  # Require all core classes.
-  require 'treat/core'
-  # Require all entity classes.
-  require 'treat/entities'
-  # Lazy load worker classes.
-  require 'treat/workers'
-  # Require proxies last.
-  require 'treat/proxies'
-  
-  # Turn sugar on.
-  Treat::Config.sweeten!
-  
-  # Install packages for a given language.
-  def self.install(language = :english)
-    require 'treat/installer'
-    Treat::Installer.install(language)
-  end
+  # * Load all the core classes. * #
+  require_relative 'treat/version'
+  require_relative 'treat/exception'
+  require_relative 'treat/autoload'
+  require_relative 'treat/modules'
 
 end

data/models/MANIFEST

@@ -0,0 +1 @@
+This is a folder containing the data models used by Treat.

data/spec/core/data_set.rb

@@ -0,0 +1,174 @@
+describe Treat::Learning::DataSet do
+
+  before do
+    @question = Treat::Learning::Question.new(:is_key_sentence, :sentence, 0, :continuous)
+    @feature = Treat::Learning::Feature.new(:word_count, 0)
+    @problem = Treat::Learning::Problem.new(@question, @feature)
+    @tag = Treat::Learning::Tag.new(:paragraph_length, 0,
+    "->(e) { e.parent_paragraph.word_count }")
+    @paragraph = Treat::Entities::Paragraph.new(
+    "Ranga and I went to the store. Meanwhile, Ryan was sleeping.")
+    @paragraph.do :segment, :tokenize
+    @sentence = @paragraph.sentences[0]
+    @data_set = Treat::Learning::DataSet.new(@problem)
+  end
+
+  describe "#initialize" do
+    context "when supplied with a problem" do
+      it "should initialize an empty data set" do
+        data_set = Treat::Learning::DataSet.new(@problem)
+        data_set.items.should eql []
+        data_set.problem.should eql @problem
+      end
+    end
+    context "when supplied with an improper argument" do
+      it "should raise an error" do
+        # The argument to initialize should be a Problem.
+        expect { data_set = Treat::Learning::DataSet.new("foo") }.to raise_error
+      end
+    end
+  end
+
+  describe "#self.build" do
+
+  end
+
+  describe "#==(other_data_set)" do
+    context "when supplied with an equivalent data set" do
+      it "returns true" do
+        data_set1 = Treat::Learning::DataSet.new(@problem)
+        data_set2 = Treat::Learning::DataSet.new(@problem)
+        data_set1.should == data_set2
+        data_set1 << @sentence
+        data_set2 << @sentence
+        data_set1.should == data_set2
+      end
+    end
+
+    context "when supplied with a non-equivalent data set" do
+      it "returns false" do
+        # Get two slightly different problems.
+        question1 = Treat::Learning::Question.new(
+        :is_key_sentence, :sentence, 0,  :continuous)
+        question2 = Treat::Learning::Question.new(
+        :is_key_word, :sentence, 0, :continuous)
+        problem1 = Treat::Learning::Problem.new(question1, @feature)
+        problem2 = Treat::Learning::Problem.new(question2, @feature)
+        # Then the problems shouldn't be equal anymore.
+        problem1.should_not == problem2
+        # Create data sets with the different problems.
+        data_set1 = Treat::Learning::DataSet.new(problem1)
+        data_set2 = Treat::Learning::DataSet.new(problem2)
+        # Then the data sets shouldn't be equal anymore.
+        data_set1.should_not == data_set2
+        # Create two data sets with the same problems.
+        data_set1 = Treat::Learning::DataSet.new(@problem)
+        data_set2 = Treat::Learning::DataSet.new(@problem)
+        # Then these should be equal.
+        data_set1.should == data_set2
+        # But when different items are added
+        data_set1 << Treat::Entities::Sentence.new(
+        "This sentence is not the same as the other.").tokenize
+        data_set2 << Treat::Entities::Sentence.new(
+        "This sentence is similar to the other.").tokenize
+        # They shouldn't be equal anymore.
+        data_set1.should_not == data_set2
+      end
+    end
+
+  end
+
+  describe "#merge" do
+    context "when supplied with two data sets refering to the same problem" do
+      it "merges the two together" do
+        # Create two data sets with the same problem.
+        data_set1 = Treat::Learning::DataSet.new(@problem)
+        data_set2 = Treat::Learning::DataSet.new(@problem)
+        # Add a sentence to each data set.
+        data_set1 << Treat::Entities::Sentence.new(
+        "This sentence is not the same as the other.").tokenize
+        data_set2 << Treat::Entities::Sentence.new(
+        "This sentence is similar to the other.").tokenize
+        # Merge the two data sets together.
+        data_set1.merge(data_set2)
+        # Check if the merge has occured properly.
+        data_set1.items.size.should eql 2
+        data_set1.items[1].should eql data_set2.items[0]
+      end
+    end
+    
+    context "when supplied with two data sets refering to different problems" do
+      it "raises an error" do
+        # Get two slightly different questions.
+        question1 = Treat::Learning::Question.new(
+        :is_key_sentence, :sentence, 0, :continuous)
+        question2 = Treat::Learning::Question.new(
+        :is_key_word, :sentence, 0, :continuous)
+        # Create two problems with the different questions.
+        problem1 = Treat::Learning::Problem.new(question1, @feature)
+        problem2 = Treat::Learning::Problem.new(question2, @feature)
+        # Create two data sets with the different problems.
+        data_set1 = Treat::Learning::DataSet.new(problem1)
+        data_set2 = Treat::Learning::DataSet.new(problem2)
+        # Add elements to each of the data sets.
+        data_set1 << Treat::Entities::Sentence.new(
+        "This sentence is not the same as the other.").tokenize
+        data_set2 << Treat::Entities::Sentence.new(
+        "This sentence is similar to the other.").tokenize
+        # Try to merge them; but this should fail.
+        expect { data_set1.merge(data_set2) }.to raise_error
+      end
+    end
+  end
+
+  describe "#<<(entity)" do
+    context "when supplied with a proper entity" do
+      it "exports the features and tags and adds them to the data set" do
+        problem = Treat::Learning::Problem.new(@question, @feature, @tag)
+        data_set = Treat::Learning::DataSet.new(problem)
+        data_set << @sentence
+        data_set.items.tap { |e| e[0][:id] = 0 }.
+        should eql [{:tags=>[11], :features=>[7, 0], :id=>0}]
+      end
+    end
+  end
+
+  describe "#serialize" do
+    context "when asked to use a given adapter" do
+      it "calls the corresponding #to_something method" do
+        
+      end
+    end
+  end
+
+  describe "#to_marshal, #self.from_marshal" do
+    context "when asked to successively serialize and deserialize data" do
+      it "completes a round trip without losing information" do
+        problem = Treat::Learning::Problem.new(@question, @feature, @tag)
+        data_set = Treat::Learning::DataSet.new(problem)
+        data_set << @sentence
+        data_set.to_marshal(file: 'test.dump')
+        Treat::Learning::DataSet.from_marshal(
+        file: 'test.dump').should == data_set
+        FileUtils.rm('test.dump')
+      end
+    end
+  end
+
+  describe "#to_mongo" do
+    
+  end
+
+  describe "#self.unserialize" do
+    context "when asked to use a given adapter" do
+      it "calls the corresponding #to_something method" do
+        
+      end
+    end
+  end
+
+  describe "#self.from_mongo" do
+
+  end
+
+end

data/spec/core/export.rb

@@ -0,0 +1,52 @@
+describe Treat::Learning::Export do
+
+  describe "#initialize" do
+    context "when supplied with acceptable parameters" do
+      it "should give access to the parameters" do
+        export = Treat::Learning::Export.new(:name, 0, "->(e) { e }")
+        export.name.should eql :name
+        export.default.should eql 0
+        export.proc_string.should eql "->(e) { e }"
+        export.proc.should be_instance_of Proc
+        export.proc.call('x').should eql 'x'
+      end
+    end
+    context "when supplied with wrong parameters" do
+      it "should raise an exception" do
+        # First argument should be a symbol representing the name of the export.
+        expect { Treat::Learning::Export.new(nil) }.to raise_error
+        # Third argument, if supplied, should be a string that
+        # evaluates to a proc (NOT a proc/lambda).
+        expect { Treat::Learning::Export.new(:name, 0, lambda { x } ) }.to raise_error
+        # Third argument should be proper ruby syntax.
+        expect { Treat::Learning::Export.new(:name, 0, "->(e) { ") }.to raise_error
+        # Third argument should evaluate to a proc.
+        expect { Treat::Learning::Export.new(:name, 0, "2") }.to raise_error
+      end
+    end
+  end
+
+  describe "#==(question)" do
+    context "when supplied with an equal question" do
+      it "should return true" do
+        Treat::Learning::Export.new(:name).
+        should == Treat::Learning::Export.new(:name)
+        Treat::Learning::Export.new(:name, 0).
+        should == Treat::Learning::Export.new(:name, 0)
+        Treat::Learning::Export.new(:name, 0, "->(e) { }").
+        should == Treat::Learning::Export.new(:name, 0, "->(e) { }")
+      end
+    end
+    context "when supplied with a different question" do
+      it "should return false" do
+        Treat::Learning::Export.new(:name).
+        should_not == Treat::Learning::Export.new(:name2)
+        Treat::Learning::Export.new(:name, 0).
+        should_not == Treat::Learning::Export.new(:name, 1)
+        Treat::Learning::Export.new(:name, 0, "->(e) { }").
+        should_not == Treat::Learning::Export.new(:name, 0, "->(e) { x }")
+      end
+    end
+  end
+
+end

data/spec/core/problem.rb

@@ -0,0 +1,144 @@
+describe Treat::Learning::Problem do
+
+  before do
+    @question = Treat::Learning::Question.new(:is_key_sentence,
+    :sentence,  0, :continuous)
+    @feature = Treat::Learning::Feature.new(:word_count, 0)
+    @tag = Treat::Learning::Tag.new(:paragraph_length, 0,
+    "->(e) { e.parent_paragraph.word_count }")
+    @paragraph = Treat::Entities::Paragraph.new(
+    "Ranga and I went to the store. Meanwhile, Ryan was sleeping.")
+    @paragraph.do :segment, :tokenize
+    @sentence = @paragraph.sentences[0]
+    @hash = {"question"=>{"name"=>:is_key_sentence, "target"=>:sentence,
+    "type"=>:continuous, "default"=>0}, "features"=>[
+    {"proc_string"=>nil, "default"=>0, "name"=>:word_count, "proc"=>nil}],
+    "tags"=>[{"proc_string"=>"->(e) { e.parent_paragraph.word_count }",
+    "default"=>0, "name"=>:paragraph_length, "proc"=>nil}], "id"=>0}
+  end
+
+  describe "#initialize" do
+    context "when supplied with proper arguments" do
+      it "initializes the problem and gives access to parameters" do
+        problem = Treat::Learning::Problem.new(@question, @feature, @tag)
+        problem.question.should eql @question
+        problem.features.should eql [@feature]
+        problem.tags.should eql [@tag]
+        problem.feature_labels.should eql [@feature.name]
+        problem.tag_labels.should eql [@tag.name]
+        # ID ???      FIXME
+      end
+    end
+    context "when supplied with unacceptable arguments" do
+      it "raises an error" do
+        # First argument should be instance of Question.
+        expect { Treat::Learning::Problem.new('foo') }.to raise_error
+        # Arguments >= 2 should be instances of Export.
+        expect { Treat::Learning::Problem.new(@question, 'foo') }.to raise_error
+        # Should have at least one Feature in the arguments.
+        expect { Treat::Learning::Problem.new(@question, @tag) }.to raise_error
+      end
+    end
+  end
+
+  describe "#==(problem)" do
+    context "when supplied with an equal problem" do
+      it "should return true" do
+        Treat::Learning::Problem.new(@question, @feature).
+        should == Treat::Learning::Problem.new(@question, @feature)
+        Treat::Learning::Problem.new(@question, @feature, @tag).
+        should == Treat::Learning::Problem.new(@question, @feature, @tag)
+      end
+    end
+    context "when supplied with a different question" do
+      it "should return false" do
+        question = Treat::Learning::Question.new(:is_key_sentence, :sentence)
+        feature = Treat::Learning::Feature.new(:word_count, 999)
+        tag = Treat::Learning::Tag.new(:paragraph_length, 999)
+        Treat::Learning::Problem.new(@question, @feature).
+        should_not == Treat::Learning::Problem.new(question, @feature)
+        Treat::Learning::Problem.new(@question, @feature).
+        should_not == Treat::Learning::Problem.new(@question, feature)
+        Treat::Learning::Problem.new(@question, @feature, @tag).
+        should_not == Treat::Learning::Problem.new(@question, @feature, tag)
+      end
+    end
+  end
+
+  describe "#export_tags(entity)" do
+    context "when called on a problem that has tags" do
+      context "and called with an entity of the proper type" do
+        it "returns an array of the tags" do
+          problem = Treat::Learning::Problem.new(@question, @feature, @tag)
+          problem.export_tags(@sentence).should eql [11]
+        end
+      end
+    end
+    context "when called on a problem that doesn't have tags" do
+      it "raises an error" do
+        problem = Treat::Learning::Problem.new(@question, @feature)
+        expect { problem.export_tags(@sentence) }.to raise_error
+      end
+    end
+  end
+
+  describe "#export_features(entity, include_answer = true)" do
+
+    context "when called with an entity of the proper type" do
+      context "and include_answer is set to true" do
+        context "and the answer is already set on the entity" do
+          it "returns an array of the exported features, with the answer" do
+            problem = Treat::Learning::Problem.new(@question, @feature)
+            @sentence.set :is_key_sentence, 1
+            problem.export_features(@sentence).should eql [7, 1]
+          end
+        end
+        context "and the answer is not already set on the entity" do
+          it "returns an array of the exported features, with the question's default answer" do
+            problem = Treat::Learning::Problem.new(@question, @feature)
+            problem.export_features(@sentence).should eql [7, @question.default]
+          end
+        end
+      end
+      context "and include_answer is set to false" do
+        it "returns an array of the exported features, without the answer" do
+          problem = Treat::Learning::Problem.new(@question, @feature)
+          problem.export_features(@sentence, false).should eql [7]
+        end
+      end
+    end
+    context "when supplied with an entity that is not of the proper type" do
+      it "raises an error" do
+        problem = Treat::Learning::Problem.new(@question, @feature)
+        word = Treat::Entities::Word.new('test')
+        expect { problem.export_features(word) }.to raise_error
+      end
+    end
+  end
+
+  describe "#to_hash" do
+    context "when called on a problem" do
+      it "returns a hash describing the problem" do
+        Treat::Learning::Problem.new(@question, @feature, @tag).
+        to_hash.tap { |e| e['id'] = 0 }.should eql @hash
+      end
+    end
+  end
+
+  describe "#self.from_hash" do
+    context "when called with a hash describing a problem" do
+      it "returns a problem based on the hash" do
+        problem = Treat::Learning::Problem.from_hash(@hash)
+        problem.question.name.should eql :is_key_sentence
+        problem.question.target.should eql :sentence
+        problem.question.type.should eql :continuous
+        problem.question.default.should eql 0
+        problem.features[0].proc_string.should eql nil
+        problem.features[0].default.should eql 0
+        problem.features[0].name.should eql :word_count
+        problem.features[0].proc.should eql nil
+      end
+    end
+  end
+
+end

data/spec/core/question.rb

@@ -0,0 +1,52 @@
+describe Treat::Learning::Question do
+
+  describe "#initialize" do
+    context "when supplied with acceptable parameters" do
+      it "should give access to the parameters" do
+        question = Treat::Learning::Question.new(
+        :is_keyword, :word, 0, :continuous)
+        question.name.should eql :is_keyword
+        question.target.should eql :word
+        question.type.should eql :continuous
+        question.default.should eql 0
+      end
+    end
+    context "when supplied with wrong parameters" do
+      it "should raise an exception" do
+        # Name should be a symbol
+        expect { Treat::Learning::Question.new(
+        nil, :sentence) }.to raise_error
+        # Target should be an actual entity type
+        expect { Treat::Learning::Question.new(
+        :name, :foo) }.to raise_error
+        # Distribution type should be continuous or discrete
+        expect { Treat::Learning::Question.new(
+        :name, :sentence, 0, :nonsense) }.to raise_error
+      end
+    end
+  end
+
+  describe "#==(question)" do
+    context "when supplied with an equal question" do
+      it "should return true" do
+        Treat::Learning::Question.new(
+        :is_keyword, :word).
+        should == Treat::Learning::Question.new(
+        :is_keyword, :word)
+      end
+    end
+    context "when supplied with a different question" do
+      it "should return false" do
+        Treat::Learning::Question.new(
+        :is_keyword, :word).
+        should_not == Treat::Learning::Question.new(
+        :is_keyword, :sentence)
+        Treat::Learning::Question.new(
+        :is_keyword, :word, 0, :continuous).
+        should_not == Treat::Learning::Question.new(
+        :is_keyword, :word, 0, :discrete)
+      end
+    end
+  end
+
+end