treat 1.2.0 → 2.0.0rc1

data/lib/treat/entities/entities.rb

@@ -0,0 +1,101 @@
+module Treat::Entities
+  
+  # * Collection and document classes * #
+  
+  # Represents a collection.
+  class Collection < Entity; end
+  
+  # Represents a document.
+  class Document < Entity; end
+    
+  # * Sections and related classes * #
+  
+  # Represents a section.
+  class Section < Entity; end
+
+  # Represents a page of text.
+  class Page < Section; end
+
+  # Represents a block of text 
+  class Block < Section; end
+  
+  # Represents a list.
+  class List < Section; end
+  
+  # * Zones and related classes * #
+  
+  # Represents a zone of text.
+  class Zone < Entity; end
+
+  # Represents a title, subtitle, 
+  # logical header of a text.
+  class Title < Zone; end
+
+  # Represents a paragraph (group 
+  # of sentences and/or phrases).
+  class Paragraph < Zone; end
+  
+  # * Groups and related classes * #
+  
+  # Represents a group of tokens.
+  class Group < Entity; end
+
+  # Represents a group of words 
+  # with a sentence ender (.!?)
+  class Sentence < Group; end
+
+  # Represents a group of words,
+  # with no sentence ender.
+  class Phrase < Group; end
+
+  # Represents a non-linguistic
+  # fragment (e.g. stray symbols).
+  class Fragment < Group; end
+  
+  # * Tokens and related classes* #
+  
+  # Represents a terminal element 
+  # (leaf) in the text structure.
+  class Token < Entity; end
+
+  # Represents a word.  Strictly,
+  # this is /^[[:alpha:]\-']+$/.
+  class Word < Token; end
+
+  # Represents an enclitic.
+  # Strictly, this is any of 
+  # 'll 'm 're 's 't or 've.
+  class Enclitic < Token; end
+
+  # Represents a number. Strictly,
+  # this is /^#?([0-9]+)(\.[0-9]+)?$/.
+  class Number < Token
+    def to_i; to_s.to_i; end
+    def to_f; to_s.to_f; end
+  end
+
+  # Represents a punctuation sign.
+  # Strictly, this is /^[[:punct:]\$]+$/.
+  class Punctuation < Token; end
+
+  # Represents a character that is neither
+  # a word, an enclitic, a number or a
+  # punctuation character (e.g. @#$%&*).
+  class Symbol < Token; end
+
+  # Represents a url. This is (imperfectly)
+  # defined as /^(http|https):\/\/[a-z0-9] 
+  # +([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}
+  # (([0-9]{1,5})?\/.*)?$/ix
+  class Url < Token; end
+
+  # Represents a valid RFC822 address.
+  # This is (imperfectly) defined as 
+  # /.+\@.+\..+/ (fixme maybe?)
+  class Email < Token; end
+  
+  # Represents a token whose type
+  # cannot be identified.
+  class Unknown; end
+  
+end

data/lib/treat/entities/{abilities/doable.rb → entity/applicable.rb}

@@ -1,8 +1,8 @@
 # Implement support for the functions #do and #do_task.
-module Treat::Entities::Abilities::Doable
+module Treat::Entities::Entity::Applicable
 
   # Perform the supplied tasks on the entity.
-  def do(*tasks)
+  def apply(*tasks)
     tasks.each do |task|
 
       if task.is_a?(Hash)
@@ -25,6 +25,8 @@ module Treat::Entities::Abilities::Doable
     end
     self
   end
+  
+  alias :do :apply
 
   # Perform an individual task on an entity
   # given a worker and options to pass to it.
@@ -33,7 +35,7 @@ module Treat::Entities::Abilities::Doable
     entity_types = group.targets
     f = nil
     entity_types.each do |t|
-      f = true if is_a?(Treat::Entities.const_get(cc(t)))
+      f = true if is_a?(Treat::Entities.const_get(t.cc))
     end
     if f || entity_types.include?(:entity)
       send(task, worker, options)

data/lib/treat/entities/{abilities → entity}/buildable.rb

@@ -3,7 +3,7 @@
 # a string or a numeric object. This class
 # is pretty much self-explanatory.
 # FIXME how can we make this language independent?
-module Treat::Entities::Abilities::Buildable
+module Treat::Entities::Entity::Buildable
 
   require 'schiphol'
   require 'fileutils'
@@ -23,23 +23,40 @@ module Treat::Entities::Abilities::Buildable
   # Build an entity from anything (can be
   # a string, numeric,folder, or file name
   # representing a raw or serialized file).
-  def build(file_or_value, options = {})
+  def build(*args)
+    
+    # This probably needs some doc.
+    if args.size == 0
+      file_or_value = ''
+    elsif args[0].is_a?(Hash)
+      file_or_value = args[0]
+    elsif args.size == 1
+      if args[0].is_a?(Treat::Entities::Entity)
+        args[0] = [args[0]]
+      end
+      file_or_value = args[0]
+    else
+      file_or_value = args
+    end
 
     fv = file_or_value.to_s
 
-    if file_or_value.is_a?(Hash)
+    if fv == ''; self.new
+    elsif file_or_value.is_a?(Array)
+      from_array(file_or_value)
+    elsif file_or_value.is_a?(Hash)
       from_db(file_or_value)
     elsif self == Treat::Entities::Document ||
       (fv.index('yml') || fv.index('yaml') ||
       fv.index('xml') || fv.index('mongo'))
       if fv =~ UriRegexp
-        from_url(fv, options)
+        from_url(fv)
       else
-        from_file(fv, options)
+        from_file(fv)
       end
     elsif self == Treat::Entities::Collection
       if FileTest.directory?(fv)
-        from_folder(fv, options)
+        from_folder(fv)
       else
         create_collection(fv)
       end
@@ -63,27 +80,35 @@ module Treat::Entities::Abilities::Buildable
   # is user-created (i.e. by calling build
   # instead of from_string directly).
   def from_string(string, enforce_type = false)
-
+    # If calling using the build syntax (i.e. user-
+    # called), enforce the type that was supplied.
     enforce_type = true if caller_method == :build
-
     unless self == Treat::Entities::Entity
       return self.new(string) if enforce_type
     end
-
     e = anything_from_string(string)
-
     if enforce_type && !e.is_a?(self)
-      raise "Asked to build a #{cl(self).downcase} "+
+      raise "Asked to build a #{self.mn.downcase} "+
       "from \"#{string}\" and to enforce type, "+
-      "but type detected was #{cl(e.class).downcase}."
+      "but type detected was #{e.class.mn.downcase}."
     end
-
     e
-
+  end
+  
+  # Build a document from an array
+  # of builders.
+  def from_array(array)
+    obj = self.new
+    array.each do |el|
+      el = el.to_entity unless el.
+      is_a?(Treat::Entities::Entity)
+      obj << el
+    end
+    obj
   end
 
   # Build a document from an URL.
-  def from_url(url, options)
+  def from_url(url)
     unless self ==
       Treat::Entities::Document
       raise Treat::Exception,
@@ -91,16 +116,22 @@ module Treat::Entities::Abilities::Buildable
       'else than a document from a url.'
     end
 
-    f = Schiphol.download(url,
-    :download_folder => Treat.paths.files,
-    :show_progress => Treat.core.verbosity.silence,
-    :rectify_extensions => true,
-    :max_tries => 3
-    )
-
-    options[:default_to] ||= 'html'
+    begin
+      folder = Treat.paths.files
+      if folder[-1] == '/'
+        folder = folder[0..-2] 
+      end
+      f = Schiphol.download(url,
+      download_folder: folder,
+      show_progress: !Treat.core.verbosity.silence,
+      rectify_extensions: true,
+      max_tries: 3)
+    rescue
+      raise Treat::Exception,
+      "Couldn't download file at #{url}."
+    end
 
-    e = from_file(f, options)
+    e = from_file(f,'html')
     e.set :url, url.to_s
     e
 
@@ -123,7 +154,7 @@ module Treat::Entities::Abilities::Buildable
 
   # Build an entity from a folder with documents.
   # Folders will be searched recursively.
-  def from_folder(folder, options)
+  def from_folder(folder)
 
     return if Reserved.include?(folder)
 
@@ -148,40 +179,48 @@ module Treat::Entities::Abilities::Buildable
 
     c = Treat::Entities::Collection.new(folder)
     folder += '/' unless folder[-1] == '/'
-
+    
+    if !FileTest.directory?(folder)
+      FileUtils.mkdir(folder)
+    end
+    
+    c.set :folder, folder
+    i = folder + '/.index'
+    c.set :index, i if FileTest.directory?(i)
+    
     Dir[folder + '*'].each do |f|
       if FileTest.directory?(f)
         c2 = Treat::Entities::Collection.
-        from_folder(f, options)
+        from_folder(f)
         c.<<(c2, false) if c2
       else
         c.<<(Treat::Entities::Document.
-        from_file(f, options), false)
+        from_file(f), false)
       end
     end
-    c
+    
+    return c
 
   end
 
   # Build a document from a raw or serialized file.
-  def from_file(file, options)
+  def from_file(file,def_fmt=nil)
 
     if file.index('yml') ||
       file.index('yaml') ||
       file.index('xml') ||
       file.index('mongo')
-      from_serialized_file(file, options)
+      from_serialized_file(file)
     else
-      fmt = Treat::Workers::Formatters::Readers::Autoselect.
-      detect_format(file, options[:default_to])
-      options[:_format] = fmt
-      from_raw_file(file, options)
+      fmt = Treat::Workers::Formatters::
+      Readers::Autoselect.detect_format(file,def_fmt)
+      from_raw_file(file, fmt)
     end
 
   end
 
   # Build a document from a raw file.
-  def from_raw_file(file, options)
+  def from_raw_file(file, def_fmt='txt')
 
     unless self ==
       Treat::Entities::Document
@@ -195,31 +234,36 @@ module Treat::Entities::Abilities::Buildable
       "Path '#{file}' does not "+
       "point to a readable file."
     end
-
-    d = Treat::Entities::Document.new(file)
-
+    options =  {default_format: def_fmt}
+    d = Treat::Entities::Document.new
+    d.set :file, file
     d.read(:autoselect, options)
 
   end
 
   # Build an entity from a serialized file.
-  def from_serialized_file(file, options)
-
-    if file.index('mongo')
-      options[:id] = file.scan(              # Consolidate this
-      /([0-9]+)\.mongo/).first.first
-      from_db(:mongo, options)
+  def from_serialized_file(file)
+    
+    unless File.readable?(file)
+      raise Treat::Exception,
+      "Path '#{file}' does not "+
+      "point to a readable file."
+    end
+    doc = Treat::Entities::Document.new
+    doc.set :file, file
+    format = nil
+    if file.index('yml') || 
+      file.index('yaml')
+      format = :yaml
+    elsif file.index('xml')
+      f = :xml
     else
-      unless File.readable?(file)
-        raise Treat::Exception,
-        "Path '#{file}' does not "+
-        "point to a readable file."
-      end
-      d = Treat::Entities::Document.new(file)
-      d.unserialize(:autoselect, options)
-      d.children[0].set_as_root!              # Fix this
-      d.children[0]
+      raise Treat::Exception,
+      "Unreadable serialized format for #{file}."
     end
+    doc.unserialize(format)
+    doc.children[0].set_as_root!              # Fix this
+    doc.children[0]
 
   end
 
@@ -238,15 +282,28 @@ module Treat::Entities::Abilities::Buildable
 
   # Build any kind of entity from a string.
   def anything_from_string(string)
+    case self.mn.downcase.intern
+    when :document
+      folder = Treat.paths.files
+      if folder[-1] == '/'
+        folder = folder[0..-2]
+      end
+      
+      now = Time.now.to_f
+      doc_file = folder+ "/#{now}.txt"
+      string.force_encoding('UTF-8')
+      File.open(doc_file, 'w') do |f|
+        f.puts string
+      end
 
-    case cl(self).downcase.intern
-    when :document, :collection
+      from_raw_file(doc_file)
+    when :collection
       raise Treat::Exception,
-      "Cannot create a document or " +
+      "Cannot create a " +
       "collection from a string " +
       "(need a readable file/folder)."
     when :phrase
-      sentence_or_phrase_from_string(string)
+      group_from_string(string)
     when :token
       token_from_string(string)
     when :zone
@@ -258,7 +315,7 @@ module Treat::Entities::Abilities::Buildable
         if string.gsub(/[\.\!\?]+/,
           '.').count('.') <= 1 &&
           string.count("\n") == 0
-          sentence_or_phrase_from_string(string)
+          group_from_string(string)
         else
           zone_from_string(string)
         end
@@ -269,15 +326,14 @@ module Treat::Entities::Abilities::Buildable
 
   end
 
+  # This should be improved on.
   def check_encoding(string)
     string.encode("UTF-8", undef: :replace) # Fix
   end
 
   # Build a phrase from a string.
-  def sentence_or_phrase_from_string(string)
-
+  def group_from_string(string)
     check_encoding(string)
-
     if !(string =~ /[a-zA-Z]+/)
       Treat::Entities::Fragment.new(string)
     elsif string.count('.!?') >= 1
@@ -285,7 +341,6 @@ module Treat::Entities::Abilities::Buildable
     else
       Treat::Entities::Phrase.new(string)
     end
-
   end
 
   # Build the right type of token

data/lib/treat/entities/{abilities → entity}/checkable.rb

@@ -1,7 +1,7 @@
 # This module implements methods that are used
 # by workers to determine if an entity is properly
 # formatted before working on it.
-module Treat::Entities::Abilities::Checkable
+module Treat::Entities::Entity::Checkable
 
   # Check if the entity has the given feature,
   # and if so return it. If not, calculate the
@@ -15,7 +15,7 @@ module Treat::Entities::Abilities::Checkable
     g2 = Treat::Workers.lookup(feature)
 
     raise Treat::Exception,
-    "#{g1.type.to_s.capitalize} #{task} " +
+    "#{g1.type.to_s.capitalize} " +
     "requires #{g2.type} #{g2.method}."
   end
 

data/lib/treat/entities/{abilities → entity}/comparable.rb

@@ -1,21 +1,21 @@
-module Treat::Entities::Abilities::Comparable
+# Allow comparison of entity hierarchy in DOM.
+module Treat::Entities::Entity::Comparable
 
+  # Determines whether the receiving class
+  # is smaller, equal or greater in the DOM
+  # hierarchy compared to the supplied one.
   def compare_with(klass)
-    
     i = 0; rank_a = nil; rank_b = nil
-    
     Treat.core.entities.order.each do |type|
-      klass2 = Treat::Entities.const_get(cc(type))
+      klass2 = Treat::Entities.const_get(type.cc)
       rank_a = i if self <= klass2
       rank_b = i if klass <= klass2
       next if rank_a && rank_b
       i += 1
     end
-    
     return -1 if rank_a < rank_b
     return 0 if rank_a == rank_b
     return 1 if rank_a > rank_b
-    
   end
 
 end

data/lib/treat/entities/{abilities → entity}/countable.rb

@@ -1,4 +1,4 @@
-module Treat::Entities::Abilities::Countable
+module Treat::Entities::Entity::Countable
 
   # Find the position of the current entity
   # inside the parent entity, starting at 1.
@@ -41,6 +41,7 @@ module Treat::Entities::Abilities::Countable
   # Returns the frequency of the given value
   # in the this entity.
   def frequency_of(value)
+    value = value.downcase
     if is_a?(Treat::Entities::Token)
       raise Treat::Exception,
       "Cannot get the frequency " +

data/lib/treat/entities/entity/debuggable.rb

@@ -0,0 +1,86 @@
+# When Treat.debug is set to true, each call to
+# #call_worker will result in a debug message being
+# printed by the #print_debug function.
+module Treat::Entities::Entity::Debuggable
+  
+  # Previous state and counter.
+  @@prev, @@i = nil, 0
+
+  # Explains what Treat is currently doing.
+  # Fixme: last call will never get shown.
+  def print_debug(entity, task, worker, group, options)
+    # Get a list of the worker's targets.
+    targets = group.targets.map(&:to_s)
+    
+    # List the worker's targets as either
+    # a single target or an and/or form
+    # (since it would be too costly to
+    # actually determine what target types
+    # were processed at runtime for each call).
+    t = targets.size == 1 ? targets[0] : targets[
+    0..-2].join(', ') + ' and/or ' + targets[-1]
+    
+    # Add genitive for annotations (sing./plural)
+    genitive = targets.size > 1 ? 'their' : 'its'
+    
+    # Set up an empty string and humanize task name.
+    doing, human_task = '', task.to_s.gsub('_', ' ')
+
+    # Base is "{task}-ed {a(n)|N} {target(s)}"
+    if [:transformer, :computer].include?(group.type)
+      tt = human_task
+      tt = tt[0..-2] if tt[-1] == 'e'
+      ed = tt[-1] == 'd' ? '' : 'ed'
+      doing = "#{tt.capitalize}#{ed} #{t}"
+    # Base is "Annotated {a(n)|N} {target(s)}"
+    elsif group.type == :annotator
+      if group.preset_option
+        opt = options[group.preset_option]
+        form = opt.to_s.gsub('_', ' ')
+        human_task[-1] = ''
+        human_task = form + ' ' + human_task
+      end
+      doing = "Annotated #{t} with " +
+      "#{genitive} #{human_task}"
+    end
+    
+    # Form is '{base} in format {worker}'.
+    if group.to_s.index('Formatters')
+      curr = doing + ' in format ' + worker.to_s
+    # Form is '{base} using {worker}'.
+    else
+      curr = doing + ' using ' + worker.to_s.gsub('_', ' ')
+    end
+    
+    # Remove any double pluralization that may happen.
+    curr.gsub!('ss', 's') unless curr.index('class')
+    
+    # Accumulate repeated tasks.
+    @@i += 1 if curr == @@prev
+    
+    # Change tasks, so output.
+    if curr != @@prev && @@prev
+      # Pluralize entity names if necessary.
+      if @@i > 1
+        Treat.core.entities.list.each do |e|
+          @@prev.gsub!(e.to_s, e.to_s + 's')
+        end
+        @@prev.gsub!('its', 'their')
+        @@prev = @@prev.split(' ').
+        insert(1, @@i.to_s).join(' ')
+      # Add determiner if singular.
+      else
+        @@prev = @@prev.split(' ').
+        insert(1, 'a').join(' ')
+      end
+      # Reset counter.
+      @@i = 0
+      # Write to stdout.
+      puts @@prev + '.'
+    end
+    
+    @@prev = curr
+  
+  end
+  
+end

data/lib/treat/entities/{abilities → entity}/delegatable.rb

@@ -1,7 +1,7 @@
 # Makes a class delegatable, allowing calls
 # on it to be forwarded to a worker class
 # able to perform the appropriate task.
-module Treat::Entities::Abilities::Delegatable
+module Treat::Entities::Entity::Delegatable
 
   # Add preset methods to an entity class.
   def add_presets(group)
@@ -10,27 +10,25 @@ module Treat::Entities::Abilities::Delegatable
     return unless opt
 
     self.class_eval do
-    group.presets.each do |preset|
-      define_method(preset) do |worker=nil, options={}|
-        return get(preset) if has?(preset)
-        options = {opt => preset}.merge(options)
-        m = group.method
-        send(m, worker, options)
-        f = unset(m)
-        features[preset] = f if f
+      group.presets.each do |preset|
+        define_method(preset) do |worker=nil, options={}|
+          return get(preset) if has?(preset)
+          options = {opt => preset}.merge(options)
+          m = group.method
+          send(m, worker, options)
+          f = unset(m)
+          features[preset] = f if f
+        end
       end
     end
-  end
 
   end
 
   # Add the workers to perform a task on an entity class.
   def add_workers(group)
     self.class_eval do
-      
       task = group.method
       add_presets(group)
-      
       define_method(task) do |worker=nil, options={}|
         if worker.is_a?(Hash)
           options, worker =
@@ -64,7 +62,7 @@ module Treat::Entities::Abilities::Delegatable
       worker_not_found(worker, group)
     end
 
-    worker = group.const_get(cc(worker.to_s).intern)
+    worker = group.const_get(worker.to_s.cc.intern)
     result = worker.send(group.method, entity, options)
 
     if group.type == :annotator && result
@@ -90,40 +88,32 @@ module Treat::Entities::Abilities::Delegatable
   # Get the default worker for that language
   # inside the given group.
   def find_worker_for_language(language, group)
-
     lang = Treat.languages[language]
     cat = group.to_s.split('::')[2].downcase.intern
-    group = ucc(cl(group)).intern
-
+    group = group.mn.ucc.intern
     if lang.nil?
       raise Treat::Exception,
       "No configuration file loaded for language #{language}."
     end
-    
     workers = lang.workers
-    
     if !workers.respond_to?(cat) ||
        !workers[cat].respond_to?(group)
         workers = Treat.languages.agnostic.workers
     end
-    
     if !workers.respond_to?(cat) || 
        !workers[cat].respond_to?(group)
       raise Treat::Exception,
       "No #{group} is/are available for the " +
       "#{language.to_s.capitalize} language."
     end
-  
-    
     workers[cat][group].first
-
   end
 
   # Return an error message and suggest possible typos.
-  def worker_not_found(klass, group)
-    "Algorithm '#{ucc(cl(klass))}' couldn't be "+
-    "found in group #{group}." + did_you_mean?(
-    group.list.map { |c| ucc(c) }, ucc(klass))
+  def worker_not_found(worker, group)
+    "Worker with name '#{worker}' couldn't be "+
+    "found in group #{group}." + Treat::Helpers::Help.
+    did_you_mean?(group.list.map { |c| c.ucc }, worker)
   end
 
 end