treat 0.2.5 → 1.0.0

data/lib/treat/processors/tokenizers/stanford.rb

@@ -1,40 +1,41 @@
-module Treat
-  module Processors
-    module Tokenizers
-      # A wrapper for the Stanford parser's Penn-Treebank
-      # style tokenizer.
-      class Stanford
-        require 'stanford-core-nlp'
-        DefaultOptions = {
-          :silence => false, 
-          :log_to_file => nil
-        }
-        @@tokenizer = nil
-        # Tokenize the entity using a Penn-Treebank style tokenizer
-        # included with the Stanford Parser.
-        #
-        # Options:
-        # - (String) :log_to_file => a filename to log output to
-        # instead of displaying it.
-        def self.tokenize(entity, options = {})
-          options = DefaultOptions.merge(options)
-          options[:log_to_file] = '/dev/null' if options[:silence]
-          if options[:log_to_file]
-            ::StanfordCoreNLP.log_file = options[:log_to_file] 
-          end
-          @@tokenizer ||= ::StanfordCoreNLP.load(:tokenize)
-          text = ::StanfordCoreNLP::Text.new(entity.to_s)
-          @@tokenizer.annotate(text)
-          text.get(:tokens).each do |token|
-            t = Treat::Entities::Token.from_string(token.value)
-            entity << t
-            t.set :character_offset_begin, 
-            token.get(:character_offset_begin)
-            t.set :character_offset_end, 
-            token.get(:character_offset_end)
-          end
-        end
-      end
+# A wrapper for the Stanford parser's
+# Penn-Treebank style tokenizer.
+class Treat::Processors::Tokenizers::Stanford
+
+  require 'treat/loaders/stanford'
+
+  @@tokenizer = nil
+
+  # Tokenize the entity using a Penn-Treebank
+  # style tokenizer.
+  #
+  # Options: none.
+  def self.tokenize(entity, options = {})
+
+    entity.check_hasnt_children
+
+    s = entity.to_s
+    
+    @@tokenizer ||=
+    ::StanfordCoreNLP.load(:tokenize)
+    text =
+    ::StanfordCoreNLP::Text.new(s)
+    @@tokenizer.annotate(text)
+
+    add_tokens(entity, text.get(:tokens))
+
+  end
+
+  # Add the tokens to the entity.
+  def self.add_tokens(entity, tokens)
+    tokens.each do |token|
+      val = token.value
+      val = '(' if val == '-LRB-'     # Fix for other special chars
+      val = ')' if val == '-RRB'
+      t = Treat::Entities::Token.
+      from_string(token.value)
+      entity << t
     end
   end
-end
+
+end

data/lib/treat/processors/tokenizers/tactful.rb

@@ -1,58 +1,67 @@
-module Treat
-  module Processors
-    module Tokenizers
-      # A tokenizer class lifted from the 'tactful-tokenizer' gem.
-      #
-      # Copyright © 2010 Matthew Bunday. All rights reserved. 
-      # Released under the GNU GPL v3. Modified by Louis Mullie.
-      # 
-      # Project website: https://github.com/SlyShy/Tactful_Tokenizer
-      class Tactful
-        ReTokenize = [
-          # Uniform Quotes
-          [/''|``/, '"'],
-          # Separate punctuation from words.
-          [/(^|\s)(')/, '\1\2'],
-          [/(?=[\("`{\[:;&#*@\.])(.)/, '\1 '],
-          [/(.)(?=[?!\)";}\]*:@\.'])|(?=[\)}\]])(.)|(.)(?=[({\[])|((^|\s)-)(?=[^-])/, '\1 '],
-          # Treat double-hyphen as a single token.
-          [/([^-])(--+)([^-])/, '\1 \2 \3'],
-          [/(\s|^)(,)(?=(\S))/, '\1\2 '],
-          # Only separate a comma if a space follows.
-          [/(.)(,)(\s|$)/, '\1 \2\3'],
-          # Combine dots separated by whitespace to be a single token.
-          [/\.\s\.\s\./, '...'],
-          # Separate "No.6"
-          [/([\W]\.)(\d+)/, '\1 \2'],
-          # Separate words from ellipses
-          [/([^\.]|^)(\.{2,})(.?)/, '\1 \2 \3'],
-          [/(^|\s)(\.{2,})([^\.\s])/, '\1\2 \3'],
-          [/(^|\s)(\.{2,})([^\.\s])/, '\1 \2\3'],
-          ##### Some additional fixes.
-          # Fix %, $, &
-          [/(\d)%/, '\1 %'],
-          [/\$(\.?\d)/, '$ \1'],
-          [/(\W)& (\W)/, '\1&\2'],
-          [/(\W\W+)&(\W\W+)/, '\1 & \2'],
-          # Fix (n 't) -> ( n't)
-          [/n 't( |$)/, " n't\\1"],
-          [/N 'T( |$)/, " N'T\\1"],
-          # Treebank tokenizer special words
-          [/([Cc])annot/, '\1an not']
-        ]
-        # Tokenize the entity using a rule-based algorithm
-        # that has been lifted from the 'tactful-tokenizer'
-        # gem.
-        def self.tokenize(entity, options = {})
-          s = entity.to_s
-          ReTokenize.each do |rules| 
-            s.gsub!(rules[0], rules[1])
-          end
-          s.split(' ').each do |token|
-            entity << Entities::Token.from_string(token)
-          end
-        end
-      end
+# A tokenizer class lifted from the 'tactful-tokenizer' gem.
+#
+# Copyright © 2010 Matthew Bunday. All rights reserved.
+# Released under the GNU GPL v3. Modified by Louis Mullie.
+#
+# Project website: https://github.com/SlyShy/Tactful_Tokenizer
+class Treat::Processors::Tokenizers::Tactful
+
+  require 'treat/helpers/decimal_point_escaper'
+  
+  ReTokenize = [
+    # Uniform Quotes
+    [/''|``/, '"'],
+    # Separate punctuation from words.
+    [/(^|\s)(')/, '\1\2'],
+    [/(?=[\("`{\[:;&#*@\.])(.)/, '\1 '],
+    [/(.)(?=[?!\)";}\]*:@\.'])|(?=[\)}\]])(.)|(.)(?=[({\[])|((^|\s)-)(?=[^-])/, '\1 '],
+    # Treat double-hyphen as a single token.
+    [/([^-])(--+)([^-])/, '\1 \2 \3'],
+    [/(\s|^)(,)(?=(\S))/, '\1\2 '],
+    # Only separate a comma if a space follows.
+    [/(.)(,)(\s|$)/, '\1 \2\3'],
+    # Combine dots separated by whitespace to be a single token.
+    [/\.\s\.\s\./, '...'],
+    # Separate "No.6"
+    [/([\W]\.)(\d+)/, '\1 \2'],
+    # Separate words from ellipses
+    [/([^\.]|^)(\.{2,})(.?)/, '\1 \2 \3'],
+    [/(^|\s)(\.{2,})([^\.\s])/, '\1\2 \3'],
+    [/(^|\s)(\.{2,})([^\.\s])/, '\1 \2\3'],
+    ##### Some additional fixes.
+    # Fix %, $, &
+    [/(\d)%/, '\1 %'],
+    [/\$(\.?\d)/, '$ \1'],
+    [/(\W)& (\W)/, '\1&\2'],
+    [/(\W\W+)&(\W\W+)/, '\1 & \2'],
+    # Fix (n 't) -> ( n't)
+    [/n 't( |$)/, " n't\\1"],
+    [/N 'T( |$)/, " N'T\\1"],
+    # Treebank tokenizer special words
+    [/([Cc])annot/, '\1an not']
+
+  ]
+
+
+  # Tokenize the entity using a rule-based algorithm
+  # that has been lifted from the 'tactful-tokenizer'
+  # gem.
+  def self.tokenize(entity, options = {})
+    
+    entity.check_hasnt_children
+    
+    s = entity.to_s
+    Treat::Helpers::DecimalPointEscaper.escape!(s)
+    
+    ReTokenize.each do |rules|
+      s.gsub!(rules[0], rules[1])
     end
+    
+    s.split(' ').each do |token|
+      entity << Treat::Entities::Token.
+      from_string(token)
+    end
+    
   end
+
 end

data/lib/treat/proxies.rb

@@ -1,40 +1,57 @@
-module Treat
-  # Proxies install Treat functions on core Ruby classes.
-  module Proxies
-    # The module proxy provides functionanaty common
-    # to the different types of proxies.
-    module Proxy
-      # Build the entity corresponding to the proxied
-      # object and send the method call to the entity.
-      def method_missing(sym, *args, &block)
-        if sym == :do || Treat::Categories.lookup(sym)
-          to_entity.send(sym, *args)
-        else
-          super(sym, *args, &block)
-        end
-      end
-      def to_entity(builder = nil)
-        Treat::Entities::Unknown(self.to_s)
+# Proxies install builders on core Ruby objects,
+# so that methods called on them may be passed
+# to the entity that can be built from the core
+# class instance.
+module Treat::Proxies
+  
+  # Provides a base functionality for proxies.
+  module Proxy
+    
+    # Build the entity corresponding to the proxied
+    # object and send the method call to the entity.
+    def method_missing(sym, *args, &block)
+      if sym == :do || Treat::Categories.lookup(sym)
+        to_entity.send(sym, *args)
+      else
+        super(sym, *args, &block)
       end
     end
-    # Install Treat functions on String objects.
-    module String
-      include Treat::Proxies::Proxy
-      # Return the entity corresponding to the string.
-      def to_entity
-        Treat::Entities::Entity.from_string(self.to_s)  
-      end
+    
+    # Create an unknown type of entity by default.
+    def to_entity(builder = nil)
+      Treat::Entities::Unknown(self.to_s)
     end
-    # Install Treat functions on Numeric objects.
-    module Numeric
-      include Treat::Proxies::Proxy
-      # Return the entity corresponding to the number.
-      def to_entity(builder = nil)
-        Treat::Entities::Number.from_numeric(self)
-      end
+    
+  end
+  
+  # Install Treat functions on String objects.
+  module String
+    
+    # Include base proxy functionality.
+    include Treat::Proxies::Proxy
+    
+    # Return the entity corresponding to the string.
+    def to_entity
+      Treat::Entities::Entity.from_string(self.to_s)
+    end
+    
+  end
+  
+  # Install Treat functions on Numeric objects.
+  module Numeric
+    
+    # Include base proxy functionality.
+    include Treat::Proxies::Proxy
+    
+    # Return the entity corresponding to the number.
+    def to_entity(builder = nil)
+      Treat::Entities::Number.from_numeric(self)
     end
-    # Include the proxies in the core classes.
-    ::String.class_eval { include Treat::Proxies::String }
-    ::Numeric.class_eval { include Treat::Proxies::Numeric }
+  
   end
-end
+  
+  # Include the proxies in the core classes.
+  ::String.class_eval { include Treat::Proxies::String }
+  ::Numeric.class_eval { include Treat::Proxies::Numeric }
+  
+end

data/lib/treat/retrievers.rb

@@ -1,17 +1,27 @@
-module Treat
-  module Retrievers
-    module Indexers
-      extend Group
-      self.type = :annotator
-      self.targets = [:collection]
-      self.default = :ferret
-    end
-    module Searchers
-      extend Group
-      self.type = :computer
-      self.targets = [:entity]
-      self.default = :ferret
-    end
-    extend Treat::Category
+# Retrievers find documents in collections.
+module Treat::Retrievers
+  
+  # Indexers create an index of words used
+  # in the documents within a collection.
+  module Indexers
+    extend Treat::Groupable
+    self.type = :annotator
+    self.targets = [:collection]
+    self.default = :ferret
   end
-end
+  
+  # Searchers perform full-text search
+  # on indexed collections in order
+  # to retrieve documents matching
+  # a query.
+  module Searchers
+    extend Treat::Groupable
+    self.type = :computer
+    self.targets = [:collection]
+    self.default = :ferret
+  end
+  
+  # Make Retrievers categorizable.
+  extend Treat::Categorizable
+  
+end

data/lib/treat/retrievers/indexers/ferret.rb

@@ -1,28 +1,49 @@
-module Treat
-  module Retrievers
-    module Indexers
-      class Ferret
-        silence_warnings { require 'ferret' }
-        require 'find'
-        require 'fileutils'
-        # Create a Ferret index for the collection and
-        # store the path to the index under "folder."
-        def self.index(collection, options = {})
-          path = "#{collection.folder}/.index"
-          FileUtils.mkdir(path) unless File.readable?(path)
-          index = ::Ferret::Index::Index.new(
-            :default_field => 'content', 
-            :path => path
-          )
-          collection.each_document do |doc|
-            index.add_document(
-              :file => doc.file, 
-              :content => doc.to_s
-            )
-          end
-          path
-        end
-      end
+# A wrapper for the indexing functions of Ferret,
+# a port of the Java Lucene search engine.
+# 
+# Documentation: 
+# http://rubydoc.info/gems/ferret
+class Treat::Retrievers::Indexers::Ferret
+  
+  # Require Ferret and file utilities.
+  silence_warnings { require 'ferret' }
+  require 'find'
+  require 'fileutils'
+  
+  # Create a Ferret index for the collection and
+  # store the index in the collection, under the
+  # path collection-folder/.index
+  # 
+  # Annotates the collection with the path to the
+  # index for future use (e.g. in searching).
+  def self.index(collection, options = {})
+    
+    path = "#{collection.folder}/.index"
+    return path if FileTest.directory?(path)
+    
+    begin
+      FileUtils.mkdir(path)
+    rescue Exception => e
+      raise Treat::Exception,
+      "Could not create folder for index " +
+      "under the collection's folder. " +
+      "(#{e.message})."
     end
+    
+    index = ::Ferret::Index::Index.new(
+      :default_field => 'content',
+      :path => path
+    )
+    
+    collection.each_document do |doc|
+      index.add_document(
+        :file => doc.file,
+        :content => doc.to_s
+      )
+    end
+    
+    path
+    
   end
-end
+  
+end

data/lib/treat/retrievers/searchers/ferret.rb

@@ -1,53 +1,72 @@
-module Treat
-  module Retrievers
-    module Searchers
-      class Ferret
-        silence_warnings { require 'ferret' }
-        require 'find'
-        DefaultOptions = {
-          :q => nil,
-          :limit => :all,
-          :callback => nil
-        }
-        # Returns an array of retrieved documents.
-        #
-        # Options:
-        #
-        # - (String) :q => a search query.
-        # - (Symbol) :limit => number of documents.
-        def self.search(collection, options = {})
-          options = DefaultOptions.merge(options)
-          unless collection.has?(:index) && collection.index
-            raise Treat::Exception, 'This collection has not been indexed.'
-          end
-          unless options[:q]
-            raise Treat::Exception, 
-            'You must set a query by using the :q option.'
-          end
-          path = "#{collection.folder}/.index"
-          unless File.readable?(path)
-            raise Treat::Exception, "The index at location #{path} cannot be found."
-          end
-          index = ::Ferret::Index::Index.new(
-            :default_field => 'content', 
-            :path => path
-          )
-          query = options.delete(:q)
-          files = {}
-          index.search_each(query, options) do |doc, score|
-            files[index[doc]['file']] = score
-          end
-          docs = []
-          files.each do |doc, score|
-            doc2 = collection.document_with_file(doc)
-            if options[:callback]
-              options[:callback].call(doc2, score)
-            end
-            docs << doc2
-          end
-          docs
-        end
+# A simple interface to the Ferret information
+# retrieval library, which performs full-text
+# search within documents of a collection.
+#
+# Documentation: 
+# http://rubydoc.info/gems/ferret
+class Treat::Retrievers::Searchers::Ferret
+  
+  silence_warnings { require 'ferret' }
+  require 'find'
+  
+  DefaultOptions = {
+    :q => nil,
+    :limit => :all,
+    :callback => nil
+  }
+  
+  # Returns an array of retrieved documents.
+  #
+  # Options:
+  #
+  # - (String) :q => a search query.
+  # - (Symbol) :limit => number of documents.
+  def self.search(collection, options = {})
+    
+    options = DefaultOptions.merge(options)
+    
+    unless collection.has?(:index)
+      raise Treat::Exception,
+      "This collection must be indexed to be searchable."
+    end
+    
+    unless options[:q]
+      raise Treat::Exception,
+      'You must set a query by using the :q option.'
+    end
+    
+    path = collection.index
+    
+    unless FileTest.directory?(path)
+      raise Treat::Exception,
+      "The index at location #{path} cannot be found."
+    end
+
+    index = ::Ferret::Index::Index.new(
+      :default_field => 'content',
+      :path => path
+    )
+    
+    query = options.delete(:q)
+    files = {}
+    index.search_each(query, options) do |doc, score|
+      files[index[doc]['file']] = score
+    end
+    
+    docs = []
+    files.each do |doc, score|
+      doc2 = collection.document_with_file(doc)
+      unless doc2
+        raise Treat::Exception,
+        "Couldn't retrieve indexed " +
+        "document with filename #{doc}."
+      end
+      if options[:callback]
+        options[:callback].call(doc2, score)
       end
+      docs << doc2
     end
+    
+    docs
   end
-end
+end