treat 0.2.5 → 1.0.0

data/lib/treat/entities/abilities/magical.rb

@@ -0,0 +1,83 @@
+module Treat::Entities::Abilities::Magical
+
+  # Parse "magic methods", which allow the following
+  # syntaxes to be used (where 'word' can be replaced
+  # by any entity type, e.g. token, zone, etc.):
+  #
+  # - each_word : iterate over each entity of type word.
+  # - words: return an array of words in the entity.
+  # - word: return the first word in the entity.
+  # - word_count: return the number of words in the entity.
+  # - words_with_*(value) (where  is an arbitrary feature):
+  #   return the words that have the given feature.
+  # - word_with_*(value) : return the first word with
+  #   the feature specified by * in value.
+  #
+  # Also provides magical methods for types of words:
+  #
+  # - each_noun:
+  # - nouns:
+  # - noun:
+  # - noun_count:
+  # - nouns_with_*(value)
+  # - noun_with_*(value)
+  #
+  def magic(sym, *args)
+
+    @@entities_regexp ||= "(#{Treat::Entities.list.join('|')})"
+    @@cats_regexp ||= "(#{Treat::Linguistics::WordCategories.join('|')})"
+
+    method = sym.to_s =~ /entities/ ?
+    sym.to_s.gsub('entities', 'entitys') :
+    method = sym.to_s
+
+    if method =~ /^#{@@entities_regexp}s$/
+      entities_with_type($1.intern)
+    elsif method =~ /^#{@@entities_regexp}$/
+      first_but_warn(entities_with_type($1.intern), $1)
+    elsif method =~ /^parent_#{@@entities_regexp}$/
+      ancestor_with_type($1.intern)
+    elsif method =~ /^each_#{@@entities_regexp}$/
+      each_entity($1.intern) { |e| yield e }
+    elsif method =~ /^#{@@entities_regexp}_count$/
+      entities_with_type($1.intern).size
+    elsif method =~ /^#{@@entities_regexp}s_with_([a-z]+)$/
+      entities_with_feature($2.intern, args[0], $1.intern)
+    elsif method =~ /^#{@@entities_regexp}_with_([a-z]*)$/
+      first_but_warn(entities_with_feature(
+      $2.intern, args[0], $1.intern), $1)
+    elsif method =~ /^each_#{@@entities_regexp}_with_([a-z]*)$/
+      entities_with_feature($2.intern, args[0], 
+      $1.intern).each { |e| yield e }
+    elsif method =~ /^each_with_([a-z]*)$/
+      entities_with_feature($2.intern, 
+      args[0], $1.intern).each { |e| yield e }
+    elsif method =~ /^each_#{@@cats_regexp}$/
+      entities_with_category($1.intern
+      ).each { |e| yield e }
+    elsif method =~ /^#{@@cats_regexp}s$/
+      entities_with_category($1.intern)
+    elsif method =~ /^#{@@cats_regexp}$/
+     first_but_warn(entities_with_category($1.intern), $1)
+    elsif method =~ /^#{@@cats_regexp}_count$/
+      entities_with_category($1.intern).size
+    elsif method =~ /^#{@@cats_regexp}s_with_([a-z]*)$/
+      entities_with_feature($2.intern, args[0], $1)
+    elsif method =~ /^#{@@cats_regexp}_with_([a-z]*)$/
+      first_but_warn(entities_with_feature(
+      $2.intern, args[0], $1.intern), $1)
+    elsif method =~ /^([a-z]*)_of_first_#{@@entities_regexp}$/
+      f = send(:"#{$2}s".intern).first
+      f ? f.send($1.intern) : nil
+    elsif method =~ /^frequency_in_#{@@entities_regexp}$/
+      frequency_in($1.intern)
+      # first_word
+      # tag_of_first_verb
+      # tag_of_title
+    else
+      return :no_magic
+    end
+  end
+  
+  
+end

data/lib/treat/entities/abilities/registrable.rb

@@ -0,0 +1,74 @@
+# Registers occurences of textual values inside
+# all children entity. Useful to calculate frequency.
+module Treat::Entities::Abilities::Registrable
+
+  # Registers a token in the @registry hash.
+  def register(entity)
+    
+    if entity.is_a?(Treat::Entities::Token) ||
+      entity.is_a?(Treat::Entities::Phrase)
+      val = entity.to_s.downcase
+      @registry[:value][val] ||= 0
+      @registry[:value][val] += 1
+    end
+  
+    @registry[:id][entity.id] = true
+    @registry[:type][entity.type] ||= 0
+    @registry[:type][entity.type] += 1
+    @registry[:position][entity.id] = @count
+    @count += 1
+    
+    @parent.register(entity) if has_parent?
+    
+  end
+
+  # Backtrack up the tree to find a token registry,
+  # by default the one in the root node of any entity.
+  def registry(type = nil)
+    if has_parent? &&
+      type != self.type
+      @parent.registry(type)
+    else
+      @registry
+    end
+  end
+
+  def contains_id?(id)
+    
+    @registry[:id][id]
+    
+  end
+
+  def contains_value?(val)
+    
+    @registry[:value][val] ?
+    true : false
+    
+  end
+
+  def contains_type?(type1)
+    
+    return true if @registry[:type][type1]
+    
+    @registry[:type].each do |type2, count|
+      if Treat::Entities.
+        match_types[type1][type2]
+        return true
+      end
+    end
+    
+    false
+    
+  end
+
+  def contains_types?(types)
+    
+    types.each do |type|
+      return true if contains_type?(type)
+    end
+    
+    false
+    
+  end
+
+end

data/lib/treat/entities/abilities/stringable.rb

@@ -0,0 +1,91 @@
+# Gives entities the ability to be converted
+# to string representations (#to_string, #to_s,
+# #to_str, #inspect, #print_tree).
+module Treat::Entities::Abilities::Stringable
+
+  # Return the entity's true string value in
+  # plain text format. Non-terminal entities
+  # will normally have an empty value.
+  def to_string; @value; end
+
+  # Returns the entity's string value by
+  # imploding the value of all terminal
+  # entities in the subtree of that entity.
+  def to_s
+    @value != '' ? @value : implode.strip
+  end
+  
+  # #to_str is the same as #to_s.
+  alias :to_str :to_s
+
+  # Return a shortened value of the entity's
+  # string value using [...], with a cutoff
+  # number of words or length.
+  def short_value(max_length = 30)
+    s = to_s
+    words = s.split(' ')
+    if s.length < max_length
+      s
+    else
+      words[0..2].join(' ') + ' [...] ' +
+      words[-2..-1].join(' ')
+    end
+  end
+
+  # Return an informative string representation
+  # of the entity.
+  def inspect
+    s = "#{cl(self.class)} (#{@id.to_s})"
+    if caller_method(2) == :inspect
+      @id.to_s
+    else
+      dependencies = []
+      @dependencies.each do |dependency|
+        dependencies <<
+        "#{dependency.target}#{dependency.type}"
+      end
+      s += "  --- #{short_value.inspect}" +
+      "  ---  #{@features.inspect} " +
+      "  --- #{dependencies.inspect} "
+    end
+    s
+  end
+
+  # Print out an ASCII representation of the tree.
+  def print_tree; puts visualize(:tree); end
+  
+  # Helper method to implode the string value of the subtree.
+  def implode
+    
+    return @value.dup if !has_children?
+    
+    value = ''
+
+    each do |child|
+      
+      if child.is_a?(Treat::Entities::Section)
+        value += "\n\n"
+      end
+      
+      if child.is_a?(Treat::Entities::Token) || child.value != ''
+        if child.is_a?(Treat::Entities::Punctuation) ||
+          child.is_a?(Treat::Entities::Clitic)
+          value.strip!
+        end
+        value += child.to_s + ' '
+      else
+        value += child.implode
+      end
+      
+      if child.is_a?(Treat::Entities::Title) ||
+        child.is_a?(Treat::Entities::Paragraph)
+        value += "\n\n"
+      end
+      
+    end
+
+    value
+
+  end
+
+end

data/lib/treat/entities/entities.rb

@@ -0,0 +1,104 @@
+module Treat::Entities
+
+  # Require the generic entity lass.
+  require 'treat/entities/entity'
+
+  # Represents a collection of texts.
+  class Collection < Entity
+    
+    # Initialize the collection with a folder
+    # containing the texts of the collection.
+    def initialize(folder = nil, id = nil)
+      super('', id)
+      set :folder, folder
+      i = folder + '/.index'
+      set :index, i if FileTest.directory?(i)
+    end
+
+    # Works like the default <<, but if the
+    # file being added is a collection or a
+    # document, then copy that collection or
+    # document into this collection's folder.
+    def <<(entities, copy = true)
+      unless entities.is_a? Array
+        entities = [entities]
+      end
+      entities.each do |entity|
+        if [:document, :collection].
+          include?(entity.type) && copy
+          entity = entity.copy_into(self)
+        end
+      end
+      super(entities)
+    end
+
+  end
+
+  # Represents a document.
+  class Document < Entity
+    
+    def initialize(file = nil, id = nil)
+      super('', id)
+      set :file, file
+    end
+
+  end
+
+  # Represents a section, usually with a title
+  # and at least one paragraph.
+  class Section < Entity; end
+
+  # Represents a zone of text
+  # (Title, Paragraph, List, Quote).
+  class Zone < Entity; end
+
+  # Represents a title, subtitle, logical header.
+  class Title < Zone; end
+
+  # Represents a paragraph.
+  class Paragraph < Zone; end
+
+  # Represents a list.
+  class List < Zone; end
+
+  # Represents a group of words.
+  class Phrase < Entity; end
+
+  # Represents a group of words with a sentence ender.
+  class Sentence < Phrase; end
+
+  # Represents a terminal element in the text structure.
+  class Token < Entity
+  end
+
+  # Represents a word.
+  class Word < Token
+  end
+
+  # Represents a clitic ('s).
+  class Clitic < Token; end
+
+  # Represents a number.
+  class Number < Token
+    def to_i; to_s.to_i; end
+    def to_f; to_s.to_f; end
+  end
+
+  # Represents a punctuation sign.
+  class Punctuation < Token; end
+
+  # Represents a character that is neither
+  # alphabetical, numerical or a punctuation
+  # character (e.g. @#$%&*).
+  class Symbol < Token; end
+
+  # Represents a url.
+  class Url < Token; end
+
+  # Represents a valid RFC822 address.
+  class Email < Token; end
+
+  # Represents an entity of unknown type.
+  class Unknown; end
+
+end

data/lib/treat/entities/entity.rb

@@ -1,258 +1,135 @@
-require 'treat/tree'
-require 'treat/feature'
-require 'treat/delegatable'
-require 'treat/visitable'
-require 'treat/registrable'
-require 'treat/buildable'
-require 'treat/doable'
-require 'treat/viewable'
-require 'treat/features'
+module Treat::Entities
 
-module Treat
-  module Entities
-    class Entity < Tree::Node
-      # A Symbol representing the lowercase version of the class name.
-      attr_accessor :type
-      # Implements support for #register
-      include Registrable
-      # Implement support for #accept.
-      include Visitable
-      # Implement support for #self.add_workers
-      extend Delegatable
-      # Implement support for #self.from_*
-      extend Buildable
-      # Implement support for #do.
-      include Doable
-      # Implement support for to_s, inspect, etc.
-      include Viewable
-      # Initialize the entity with its value and
-      # (optionally) a unique identifier. By default,
-      # the object_id will be used as id. Also initialize
-      # the token registry in the root node.
-      def initialize(value = '', id = nil)
-        id ||= object_id
-        super(value, id)
-        @type = :entity
-        # @match_types = Treat::Entities.match_types
-      end
-      # Catch missing methods to support method-like
-      # access to features (e.g. entity.categoryinstead of
-      # entity.features[:cat]) and to support magic
-      # methods (see #parse_magic_method). If the
-      # feature does not exist
-      def method_missing(sym, *args, &block)
-        return self.build(*args) if sym == nil
-        if !@features.has_key?(sym)
-          r = parse_magic_method(sym, *args, &block)
-          if r == :no_magic
-            begin
-              super(sym, *args, &block)
-            rescue NoMethodError
-              return false if sym.to_s[-1] == '?'
-              if Categories.lookup(sym)
-                msg = "Method #{sym} cannot be called on a #{type}."
-              else
-                msg = "Method #{sym} does not exist."
-                msg += did_you_mean?(Category.methods, sym)
-              end
-              raise Treat::Exception, msg
-            end
-          else
-            r
-          end
-        else
-          @features[sym]
-        end
-      end
-      # Parse "magic methods", which allow the following
-      # syntaxes to be used (where 'word' can be replaced
-      # by any entity type, e.g. token, zone, etc.):
-      #
-      # - each_word : iterate over each entity of type word.
-      # - words: return an array of words in the entity.
-      # - word: return the first word in the entity.
-      # - word_count: return the number of words in the entity.
-      # - words_with_*(value) (where  is an arbitrary feature):
-      #   return the words that have the given feature.
-      # - word_with_*(value) : return the first word with
-      #   the feature specified by * in value.
-      #
-      # Also provides magical methods for types of words:
-      #
-      # - each_noun:
-      # - nouns:
-      # - noun:
-      # - noun_count:
-      # - nouns_with_*(value)
-      # - noun_with_*(value)
-      #
-      def parse_magic_method(sym, *args)
-        @@entities_regexp ||= "(#{Treat::Entities.list.join('|')})"
-        @@cats_regexp ||= "(#{Treat::Languages::WordCategories.join('|')})"
-        method = sym.to_s =~ /entities/ ?
-        sym.to_s.gsub('entities', 'entitys') :
-        method = sym.to_s
-        if method =~ /^#{@@entities_regexp}s$/
-          a = []
-          each_entity($1.intern) { |e| a << e }
-          a
-        elsif method =~ /^#{@@entities_regexp}$/
-          a = []
-          each_entity($1.intern) { |e| a << e }
-          first_but_warn(a, $1)
-        elsif method =~ /^parent_#{@@entities_regexp}$/
-          ancestor_with_types($1.intern)
-        elsif method =~ /^each_#{@@entities_regexp}$/
-          each_entity($1.intern) { |e| yield e }
-        elsif method =~ /^#{@@entities_regexp}_count$/
-          i = 0
-          each_entity($1.intern) { |e| i += 1 }
-          i
-        elsif method =~ /^#{@@entities_regexp}s_with_([a-z]+)$/
-          a = []
-          each_entity($1.intern) do |e|
-            a << e if e.has?($2.intern) &&
-            e.send($2.intern) == args[0]
-          end
-          a
-        elsif method =~ /^#{@@entities_regexp}_with_([a-z]*)$/
-          a = []
-          each_entity($1.intern) do |e|
-            a << e if e.has?($2.intern) &&
-            e.send($2.intern) == args[0]
-          end
-          first_but_warn(a, $1)
-        elsif method =~ /^each_with_([a-z]*)$/
-          each_entity do |e|
-            yield e if e.has?($1.intern) &&
-            e.send($1.intern) == args[0]
-          end
-        elsif method =~ /^each_#{@@cats_regexp}$/
-          each_entity(:word) { |e| yield e if e.category == $1.intern }
-        elsif method =~ /^#{@@cats_regexp}s$/
-          a = []
-          each_entity(:word) { |e| a << e if e.category == $1.intern }
-          a
-        elsif method =~ /^#{@@cats_regexp}$/
-          a = []
-          each_entity(:word) { |e| a << e if e.category == $1.intern }
-          first_but_warn(a, $1)
-        elsif method =~ /^#{@@cats_regexp}_count$/
-          i = 0
-          each_entity(:word) { |e| i += 1 if e.category == $1.intern }
-          i
-        elsif method =~ /^#{@@cats_regexp}s_with_([a-z]*)$/
-          a = []
-          each_entity(:word) do |e|
-            a << e if e.category == $1.intern &&
-            e.has?($2.intern) && e.send($2.intern) == args[0]
-          end
-          a
-        elsif method =~ /^#{@@cats_regexp}_with_([a-z]*)$/
-          a = []
-          each_entity(:word) do |e|
-            a << e if e.category== $1.intern &&
-            e.has?($2.intern) && e.send($2.intern) == args[0]
-          end
-          first_but_warn(a, $1)
-        elsif method =~ /^is_#{@@entities_regexp}\?$/
-          type.to_s == $1
-        elsif method =~ /^is_#{@@cats_regexp}\?$/
-          category.to_s == $1
-        else
-          return :no_magic
-        end
+  # Require base class for Entity.
+  require 'treat/tree'
+
+  class Entity < Treat::Tree::Node
+
+    # A Symbol representing the lowercase
+    # version of the class name.
+    attr_accessor :type
+
+    # Require abilities.
+    require 'treat/entities/abilities'
+
+    # Implements support for #register,
+    # #registry, and #contains_* methods.
+    include Abilities::Registrable
+
+    # Implement support for #self.add_workers
+    extend Abilities::Delegatable
+
+    # Implement support for #self.print_debug and
+    # #self.invalid_call_msg
+    extend Abilities::Debuggable
+
+    # Implement support for #self.build
+    # and #self.from_*
+    extend Abilities::Buildable
+
+    # Implement support for #do.
+    include Abilities::Doable
+
+    # Implement support for #frequency,
+    # #frequency_in_parent and #position_in_parent.
+    include Abilities::Countable
+
+    # Implement support for #magic.
+    include Abilities::Magical
+
+    # Implement support for #to_s, #inspect, etc.
+    include Abilities::Stringable
+
+    # Implement support for #check_has
+    # and #check_hasnt_children?
+    include Abilities::Checkable
+
+    # Implement support for #each_entity, as well as
+    # #entities_with_type, #ancestors_with_type,
+    # #entities_with_feature, #entities_with_category.
+    include Abilities::Iterable
+    
+    # Implement support for #export to export
+    # a line of a data set based on a classification.
+    include Abilities::Exportable
+    
+    # Implement support for #copy_into.
+    include Abilities::Copyable
+    
+    # Initialize the entity with its value and
+    # (optionally) a unique identifier. By default,
+    # the object_id will be used as id.
+    def initialize(value = '', id = nil)
+      id ||= object_id
+      super(value, id)
+      @type = :entity if self == Entity
+      @type ||= ucc(cl(self.class)).intern
+      unless is_a?(Treat::Entities::Token)
+        @count = 0
+        @registry = {
+          :id => {},
+          :value => {},
+          :type => {},
+          :position => {}
+        }
       end
-      # Add an entity to the current entity.
-      # Registers the entity in the root node
-      # token registry if the entity is a leaf.
-      #
-      # @see Treat::Registrable
-      def <<(entities, clear_parent = true)
-        entities = [entities] unless entities.is_a? Array
-        entities.each do |entity|
-          if entity.is_a?(Treat::Entities::Token) ||
-            entity.is_a?(Treat::Entities::Phrase)
-            register_token(entity) unless entity.value == ''
-          end
-        end
-        super(entities)
-        @parent.value = '' if has_parent?
-        entities[0]
+    end
+    
+    
+    # Add an entity to the current entity.
+    # Registers the entity in the root node
+    # token registry if the entity is a leaf.
+    #
+    # @see Treat::Registrable
+    def <<(entities, clear_parent = true)
+      unless entities.is_a? Array
+        entities = [entities]
       end
-      # Yields each entity of any of the supplied
-      # types in the children tree of this Entity.
-      # Note that this function is recursive, unlike
-      # #each. It does not yield the top element being
-      # recursed.
-      #
-      # This function NEEDS to be ported to C (see source).
-      def each_entity(*types)
-        types = [:entity] if types.size == 0
-        f = false
-        types.each { |t2| f = true if Treat::Entities.match_types[t2][type] }
-        yield self if f
-        unless @children.size == 0
-          @children.each do |child|
-            child.each_entity(*types) { |y| yield y }
-          end
-        end
+      entities.each do |entity|
+        register(entity)
       end
-
-      # Replace with:
-      #inline do |builder|
-        #  
-       # builder.c_raw <<-EOS, :arity => -1
+      super(entities)
+      @parent.value = '' if has_parent?
+      entities[0]
+    end
 
 
+    # Catch missing methods to support method-like
+    # access to features (e.g. entity.category
+    # instead of entity.features[:category]) and to
+    # support magic methods (see #magic).
+    #
+    # If the feature or magic method does not exist,
+    # or can't be parsed, raises an exception.
+    #
+    # Also catches the "empty" method call (e.g.
+    # Word('hello') or Word 'hello') as syntactic
+    # sugar for the #self.build method.
+    def method_missing(sym, *args, &block)
+      return self.build(*args) if sym == nil
 
-        #EOS
-      #end
-      # Returns the first ancestor of this entity that has the given type.
-      def ancestor_with_types(*types)
-        ancestor = @parent
-        match_types = lambda do |t1, t2s|
-          f = false
-          t2s.each do |t2|
-            if Treat::Entities.match_types[t1][t2]
-              f = true; break
-            end
-          end
-          f
-        end
-        if ancestor
-          while not match_types.call(ancestor.type, types)
-            return nil unless (ancestor && ancestor.has_parent?)
-            ancestor = ancestor.parent
+      if !@features.has_key?(sym)
+        r = magic(sym, *args, &block)
+        return r unless r == :no_magic
+        begin
+          super(sym, *args, &block)
+        rescue NoMethodError
+          raise Treat::Exception,
+          if Treat::Categories.lookup(sym)
+            msg = "Method #{sym} cannot " +
+            "be called on a #{type}."
+          else
+            msg = "Method #{sym} does not exist."
+            msg += did_you_mean?(
+            Treat::Categories.methods, sym)
           end
-          match_types.call(ancestor.type, types) ? ancestor : nil
-        end
-      end
-      alias :ancestor_with_type :ancestor_with_types
-      # Returns the (direct) ancestors of this entity that
-      # have the given type.
-      def ancestors_with_types(*types)
-        ancestor = self
-        ancestors = []
-        while (a = ancestor.ancestor_with_types(*types))
-          ancestors << a
-          ancestor = ancestor.parent
         end
-        ancestors
-      end
-      alias :ancestors_with_type :ancestors_with_types
-      # Return the first element in the array, warning if not
-      # the only one in the array. Used for magic methods: e.g.,
-      # the magic method "word" if called on a sentence
-      # with many words, Treat will return the first word
-      # but warn the user.
-      def first_but_warn(array, type)
-        if array.size > 1
-          warn "Warning: requested one #{type}, but" +
-          " there are many #{type}s in the given entity."
-        end
-        array[0]
+      else
+        @features[sym]
       end
+
     end
+
   end
+
 end