treat 1.0.6 → 1.1.0

data/lib/treat/{processors → workers/processors}/chunkers/html.rb

@@ -1,4 +1,4 @@
-class Treat::Processors::Chunkers::HTML
+class Treat::Workers::Processors::Chunkers::HTML
 
   require 'nokogiri'
 
@@ -24,6 +24,7 @@ class Treat::Processors::Chunkers::HTML
         node.parent && node.parent.type == :section)
         
         if $1
+          
           lvl = $1.to_i
           if lvl <= level
             node.ancestors_with_type(:section).
@@ -41,13 +42,13 @@ class Treat::Processors::Chunkers::HTML
           node.set :level, level
 
         end
-
+        
         t = node <<
         Treat::Entities::Title.new(txt)
         t.set :level, level
 
       elsif child.name == 'p'
-        
+
         node << Treat::Entities::Zone.
         from_string(txt)
 

data/lib/treat/workers/processors/chunkers/txt.rb

@@ -0,0 +1,32 @@
+class Treat::Workers::Processors::Chunkers::TXT
+  
+  # Separates a string into
+  # zones on the basis of newlines.
+  #
+  # Options: none.
+  def self.chunk(entity, options = {})
+    
+    entity.check_hasnt_children
+    zones = entity.to_s.split("\n")
+    current = entity
+    zones.each do |zone|
+      zone.strip!
+      next if zone == ''
+      c = Treat::Entities::
+      Zone.from_string(zone)
+      if c.type == :title
+        if current.type == :section
+          current = current.parent
+          current = entity << Treat::
+          Entities::Section.new
+        else
+          current = entity << Treat::
+          Entities::Section.new
+        end
+      end
+      current << c
+    end
+    
+  end
+  
+end

data/lib/treat/{processors → workers/processors}/parsers/enju.rb

@@ -10,7 +10,7 @@
 # Takuya Matsuzaki, Yusuke Miyao, and Jun'ichi Tsujii.
 # 2007. Efficient HPSG Parsing with Supertagging and
 # CFG-filtering. In Proceedings of IJCAI 2007.
-module Treat::Processors::Parsers::Enju
+module Treat::Workers::Processors::Parsers::Enju
 
   # Require the 'open3' library to connect
   # with the background Enju process.
@@ -23,10 +23,10 @@ module Treat::Processors::Parsers::Enju
   @@parser = nil
   
   # A hash of Enju cat tags mapped to word categories.
-  Ectc = Treat::Universalisation::Tags::EnjuCatToCategory
+  Ectc = Treat.tags.enju.cat_to_category
   
   # A hash of Enju cat/xcat pairs mapped to PTB tags.
-  Ecxtp = Treat::Universalisation::Tags::EnjuCatXcatToPTB
+  Ecxtp = Treat.tags.enju.xcat_to_ptb
   
   # Parse the entity into its syntactical 
   # phrases using Enju.

data/lib/treat/{processors → workers/processors}/parsers/stanford.rb

@@ -1,8 +1,10 @@
 # A wrapper class for the Stanford parser.
-class Treat::Processors::Parsers::Stanford
+class Treat::Workers::Processors::Parsers::Stanford
 
   require 'treat/loaders/stanford'
 
+  Pttc = Treat.tags.aligned.phrase_tags_to_category
+  
   # Hold one instance of the pipeline per language.
   @@parsers = {}
 
@@ -27,9 +29,7 @@ class Treat::Processors::Parsers::Stanford
     lang = entity.language
     init(lang, options)
     
-    tag_set =  Treat::Universalisation::Tags::
-               StanfordTagSetForLanguage[
-               Treat::Languages.describe(lang)]
+    tag_set = StanfordCoreNLP::Config::TagSets[language]
     
     text = ::StanfordCoreNLP::Text.new(val)
     @@parsers[lang].annotate(text)
@@ -58,8 +58,7 @@ class Treat::Processors::Parsers::Stanford
   def self.init(lang, options)
     return if @@parsers[lang]
     
-    language = Treat::Languages.describe(lang)
-    Treat::Loaders::Stanford.load(language)
+    Treat::Loaders::Stanford.load(lang)
     
     options = DefaultOptions.merge(options)
     StanfordCoreNLP.use(lang)
@@ -117,8 +116,7 @@ class Treat::Processors::Parsers::Stanford
         tag_s, tag_opt = *tag.split('-')
         tag_s ||= ''
         
-        if Treat::Universalisation::Tags::PhraseTagToCategory[tag_s] &&
-           Treat::Universalisation::Tags::PhraseTagToCategory[tag_s][tag_set]
+        if Pttc[tag_s] && Pttc[tag_s][tag_set]
           ruby_child = Treat::Entities::Phrase.new
         else
           l = java_child.children[0].to_s

data/lib/treat/{processors → workers/processors}/segmenters/punkt.rb

@@ -5,9 +5,7 @@
 # Original paper: Kiss, Tibor and Strunk, Jan (2006):
 # Unsupervised Multilingual Sentence Boundary Detection.
 # Computational Linguistics 32: 485-525.
-module Treat::Processors::Segmenters::Punkt
-
-  require 'treat/helpers/decimal_point_escaper'
+module Treat::Workers::Processors::Segmenters::Punkt
   
   # Require silently the punkt-segmenter gem.
   silence_warnings { require 'punkt-segmenter' }
@@ -41,7 +39,8 @@ module Treat::Processors::Segmenters::Punkt
     # Replace the point in all floating-point numbers
     # by ^^; this is a fix since Punkt trips on decimal 
     # numbers.
-    Treat::Helpers::DecimalPointEscaper.escape!(s)
+
+    escape_floats!(s)
     s.gsub!(/([^\.\?!]\.|\!|\?)([^\s"'])/) { $1 + ' ' + $2 }
     
     result = @@segmenters[lang].
@@ -50,8 +49,7 @@ module Treat::Processors::Segmenters::Punkt
     
     result.each do |sentence|
       # Unescape the sentence.
-      Treat::Helpers::DecimalPointEscaper.
-      unescape!(sentence)
+      unescape_floats!(sentence)
       entity << Treat::Entities::Phrase.
         from_string(sentence)
     end
@@ -65,13 +63,11 @@ module Treat::Processors::Segmenters::Punkt
     if options[:model]
       model = options[:model]
     else
-      l = Treat::Languages.describe(lang)
-      model = "#{Treat.models}punkt/#{l}.yaml"
-
+      model = "#{Treat.paths.models}punkt/#{lang}.yaml"
       unless File.readable?(model)
         raise Treat::Exception,
         "Could not get the language model " +
-        "for the Punkt segmenter for #{l}."
+        "for the Punkt segmenter for #{lang.to_s.capitalize}."
       end
     end
     

data/lib/treat/{processors → workers/processors}/segmenters/stanford.rb

@@ -1,6 +1,6 @@
 # A wrapper for the sentence splitter supplied by
 # the Stanford parser.
-class Treat::Processors::Segmenters::Stanford
+class Treat::Workers::Processors::Segmenters::Stanford
 
   require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load
@@ -41,7 +41,7 @@ class Treat::Processors::Segmenters::Stanford
       from_string(sentence, true)
       entity << s
       if options[:also_tokenize]
-        Treat::Processors::Tokenizers::Stanford.
+        Treat::Workers::Processors::Tokenizers::Stanford.
         add_tokens(s, sentence.get(:tokens))
       end
     end

data/lib/treat/{processors → workers/processors}/segmenters/tactful.rb

@@ -7,7 +7,7 @@
 # Original paper: Dan Gillick. 2009. Sentence Boundary Detection
 # and the Problem with the U.S. University of California, Berkeley.
 # http://dgillick.com/resource/sbd_naacl_2009.pdf
-module Treat::Processors::Segmenters::Tactful
+module Treat::Workers::Processors::Segmenters::Tactful
   
   # Require the 'tactful_tokenizer' gem.
   silence_warnings { require 'tactful_tokenizer' }
@@ -15,8 +15,6 @@ module Treat::Processors::Segmenters::Tactful
   # Remove function definition 'tactful_tokenizer' by gem.
   String.class_eval { undef :tokenize }
   
-  require 'treat/helpers/decimal_point_escaper'
-  
   # Keep only one copy of the segmenter.
   @@segmenter = nil
   
@@ -30,7 +28,7 @@ module Treat::Processors::Segmenters::Tactful
     
     s = entity.to_s
     
-    Treat::Helpers::DecimalPointEscaper.escape!(s)
+    escape_floats!(s)
     
     s.gsub!(/([^\.\?!]\.|\!|\?)([^\s"'])/) { $1 + ' ' + $2 }
     
@@ -39,8 +37,7 @@ module Treat::Processors::Segmenters::Tactful
     sentences = @@segmenter.tokenize_text(s)
     
     sentences.each do |sentence|
-      Treat::Helpers::DecimalPointEscaper.unescape!(sentence)
-      puts sentence.to_s if sentence.to_s.include?('staff')
+      unescape_floats!(sentence)
       entity << Treat::Entities::Phrase.from_string(sentence)
     end
     

data/lib/treat/{processors → workers/processors}/tokenizers/ptb.rb

@@ -11,7 +11,7 @@
 # All rights reserved. This program is free software;
 # you can redistribute it and/or modify it under the
 # same terms as Ruby itself.
-module Treat::Processors::Tokenizers::PTB
+module Treat::Workers::Processors::Tokenizers::PTB
   
   # Tokenize the entity using a native rule-based algorithm.
   def self.tokenize(entity, options = {})
@@ -35,14 +35,11 @@ module Treat::Processors::Tokenizers::PTB
     
     s = " " + string + " "
     
-    # Translate some common extended ascii 
-    # characters to quotes
     s.gsub!(/‘/,'`')
     s.gsub!(/’/,"'")
     s.gsub!(/“/,"``")
     s.gsub!(/”/,"''")
-    
-    
+  
     s.gsub!(/\s+/," ")
     s.gsub!(/(\s+)''/,'\1"')
     s.gsub!(/(\s+)``/,'\1"')
@@ -83,6 +80,10 @@ module Treat::Processors::Tokenizers::PTB
     s.gsub!(/\//, ' / ')
     s.gsub!(/\s+/,' ')
     s.strip!
+  
+    s.gsub!(/``/,'"')
+    s.gsub!(/''/,'"')
+    
     s.split(/\s+/)
   end
   

data/lib/treat/{processors → workers/processors}/tokenizers/punkt.rb

@@ -12,7 +12,7 @@
 # (almost rewrite).
 #
 # Project website: https://github.com/lfcipriani/punkt-segmenter
-class Treat::Processors::Tokenizers::Punkt
+class Treat::Workers::Processors::Tokenizers::Punkt
   
   SentEndChars = ['.', '?', '!']
   ReSentEndChars = /[.?!]/

data/lib/treat/{processors → workers/processors}/tokenizers/stanford.rb

@@ -1,6 +1,6 @@
 # A wrapper for the Stanford parser's
 # Penn-Treebank style tokenizer.
-class Treat::Processors::Tokenizers::Stanford
+class Treat::Workers::Processors::Tokenizers::Stanford
 
   require 'treat/loaders/stanford'
   Treat::Loaders::Stanford.load

data/lib/treat/{processors → workers/processors}/tokenizers/tactful.rb

@@ -4,10 +4,8 @@
 # Released under the GNU GPL v3. Modified by Louis Mullie.
 #
 # Project website: https://github.com/SlyShy/Tactful_Tokenizer
-class Treat::Processors::Tokenizers::Tactful
+class Treat::Workers::Processors::Tokenizers::Tactful
 
-  require 'treat/helpers/decimal_point_escaper'
-  
   ReTokenize = [
     # Uniform Quotes
     [/''|``/, '"'],
@@ -52,7 +50,7 @@ class Treat::Processors::Tokenizers::Tactful
     
     s = entity.to_s
     
-    Treat::Helpers::DecimalPointEscaper.escape!(s)
+    escape_floats!(s)
     
     ReTokenize.each do |rules|
       s.gsub!(rules[0], rules[1])
@@ -60,7 +58,7 @@ class Treat::Processors::Tokenizers::Tactful
     
     s.split(' ').each do |token|
       
-      Treat::Helpers::DecimalPointEscaper.unescape!(token)
+      unescape_floats!(token)
       entity << Treat::Entities::Token.
       from_string(token)
     end

data/lib/treat/{retrievers → workers/retrievers}/indexers/ferret.rb

@@ -3,7 +3,7 @@
 # 
 # Documentation: 
 # http://rubydoc.info/gems/ferret
-class Treat::Retrievers::Indexers::Ferret
+class Treat::Workers::Retrievers::Indexers::Ferret
   
   # Require Ferret and file utilities.
   silence_warnings { require 'ferret' }

data/lib/treat/{retrievers → workers/retrievers}/searchers/ferret.rb

@@ -4,7 +4,7 @@
 #
 # Documentation: 
 # http://rubydoc.info/gems/ferret
-class Treat::Retrievers::Searchers::Ferret
+class Treat::Workers::Retrievers::Searchers::Ferret
   
   silence_warnings { require 'ferret' }
   require 'find'

data/lib/treat/workers.rb

@@ -0,0 +1,96 @@
+# This module creates all the worker categories
+# and the groups within these categories and adds
+# the relevant hooks on the appropriate entities.
+module Treat::Workers
+
+  require 'treat/workers/group'
+
+  # A lookup table for entity types.
+  @@lookup = {}
+
+  # Find a worker group based on method.
+  def self.lookup(method)
+    @@lookup[method]
+  end
+
+  def self.create_categories
+    Treat.workers.list.each do |cat|
+      create_category(cat.to_s.
+      capitalize.intern,
+      load_category_conf(cat))
+    end
+  end
+
+  def self.load_category_conf(name)
+    config = Treat.workers[name]
+    if config.nil?
+      raise Treat::Exception,
+      "The configuration file " +
+      "for #{cat_sym} is missing."
+    end
+    config
+  end
+
+  def self.create_category(name, conf)
+    category = self.const_set(name, Module.new)
+    conf.each_pair do |group, worker|
+      name = group.to_s.capitalize.intern
+      category.module_eval do
+        @@methods = []; def methods; 
+        @@methods; end; def groups; 
+        self.constants; end
+      end
+      self.create_group(name, worker, category)
+    end
+  end
+
+  def self.create_group(name, conf, category)
+    group = category.const_set(name, Module.new)
+    self.set_group_options(group, conf)
+    self.bind_group_targets(group)
+    self.register_group_presets(group, conf)
+    @@methods << group.method
+    @@lookup[group.method] = group
+  end
+
+  def self.bind_group_targets(group)
+    group.targets.each do |entity_type|
+      entity = Treat::Entities.
+      const_get(cc(entity_type))
+      entity.class_eval do
+        add_workers group
+      end
+    end
+  end
+
+  def self.register_group_presets(group, conf)
+    return unless conf.respond_to? :presets
+    conf.presets.each do |m|
+      @@methods << m
+      @@lookup[m] = group
+    end
+  end
+
+  def self.set_group_options(group, conf)
+    group.module_eval do
+      extend Treat::Workers::Group
+      self.type = conf.type
+      self.targets = conf.targets
+      if conf.respond_to?(:default)
+        self.default = conf.default
+      end
+      if conf.respond_to?(:preset_option)
+        self.preset_option = conf.preset_option
+      end
+      if conf.respond_to?(:presets)
+        self.presets = conf.presets
+      end
+      if conf.respond_to?(:recursive)
+        self.recursive = conf.recursive
+      end
+    end
+  end
+
+  self.create_categories
+
+end

data/lib/treat.rb

@@ -1,62 +1,36 @@
 module Treat
   
-  # Require custom exception cass.
-  require 'treat/exception'
-
-  # Treat requires Ruby 1.9 or higher.
-  if RUBY_VERSION <= '1.9'
-    raise Treat::Exception,
-    'Treat requires Ruby 1.9 or higher.'
-  end
-
-  # The current version of Treat.
-  VERSION = "1.0.6"
-
-  # Add methods to handle syntactic sugar,
-  # language configuration options, and paths.
-  require 'treat/configurable'
-  extend Treat::Configurable
-
-  # The folders in the library and descriptions.
-  Paths = {
-    :tmp => 'temporary files',
-    :lib => 'class and module definitions',
-    :bin => 'binary files',
-    :files => 'user-saved files',
-    :data => 'data set files',
-    :models => 'model files',
-    :spec => 'spec test files'
-  }
-
-  # Add methods to provide access to common paths.
-  class << self
-    Paths.each do |path, _|
-      define_method(path) do
-        (File.dirname(__FILE__).
-        split('/')[0..-2].join('/') + 
-        '/' + path.to_s + '/').gsub(
-        'lib/../', '')
-      end
-    end
+  # Treat requires Ruby >= 1.9.2
+  if RUBY_VERSION < '1.9.2'
+    raise "Treat requires Ruby version 1.9.2 " +
+    "or higher, but current is #{RUBY_VERSION}."
   end
-
-  require 'treat/object'
-  require 'treat/kernel'
-  require 'treat/downloader'
-  require 'treat/languages'
-  require 'treat/universalisation'
+  
+  # Custom exception class.
+  class Exception < ::Exception; end
+  
+  # Load configuration options.
+  require 'treat/config'
+  # Load all workers.
+  require 'treat/helpers'
+  # Require library loaders.
+  require 'treat/loaders'
+  # Require all core classes.
+  require 'treat/core'
+  # Require all entity classes.
   require 'treat/entities'
-  require 'treat/categories'
-  require 'treat/data_set'
+  # Lazy load worker classes.
+  require 'treat/workers'
+  # Require proxies last.
   require 'treat/proxies'
   
+  # Turn sugar on.
+  Treat::Config.sweeten!
+  
   # Install packages for a given language.
   def self.install(language = :english)
     require 'treat/installer'
     Treat::Installer.install(language)
   end
 
-  # Enable syntactic sugar by default.
-  Treat.sweeten!
-  
 end

data/spec/collection.rb

@@ -3,7 +3,7 @@ require_relative '../lib/treat'
 describe Treat::Entities::Collection do
 
   before :all do
-    @file = Treat.spec + 'samples/mathematicians'
+    @file = Treat.paths.spec + 'samples/mathematicians'
   end
 
   describe "#<<" do
@@ -12,7 +12,7 @@ describe Treat::Entities::Collection do
 
       it "copies the document to the collection's folder " +
       "and adds the document object to the collection" do
-        f = Treat.spec + 'samples/test'
+        f = Treat.paths.spec + 'samples/test'
         ff = '3_2_release_notes.html'
         u = 'http://guides.rubyonrails.org/' + ff
         c = Treat::Entities::Collection.build(f)
@@ -26,7 +26,7 @@ describe Treat::Entities::Collection do
 
     context "when supplied with anything else" do
       it "adds the object to the collection" do
-        f = Treat.spec + 'samples/test'
+        f = Treat.paths.spec + 'samples/test'
         c = Treat::Entities::Collection.build(f)
         c << Treat::Entities::Document.new
         c.size.should eql 2
@@ -53,7 +53,7 @@ describe Treat::Entities::Collection do
       context "when supplied a folder name that doesn't exist" do
 
         it "creates the directory and opens the collection" do
-          f = Treat.spec + 'samples/test'
+          f = Treat.paths.spec + 'samples/test'
           c = Treat::Entities::Collection.build(f)
           FileTest.directory?(f).should eql true
           c.should be_an_instance_of Treat::Entities::Collection

data/spec/document.rb

@@ -8,7 +8,7 @@ describe Treat::Entities::Document do
 
       it "returns a list of general topics the document belongs to" do
         #doc = Treat::Entities::Document.new(
-        #Treat.spec + 'samples/mathematicians/archimedes.abw').read(:abw)
+        #Treat.paths.spec + 'samples/mathematicians/archimedes.abw').read(:abw)
         #doc.do(:chunk, :segment, :tokenize)
         #puts doc.topics.inspect
       end
@@ -24,7 +24,7 @@ describe Treat::Entities::Document do
       context "when supplied with a readable file name" do
         it "opens the file and reads its " +
         "content into a document" do
-          f = Treat.spec + 'samples/mathematicians/leibniz.txt'
+          f = Treat.paths.spec + 'samples/mathematicians/leibniz.txt'
           d = Treat::Entities::Document.build(f)
           d.should be_an_instance_of Treat::Entities::Document
           d.to_s.index('Gottfried Leibniz').should_not eql nil
@@ -36,7 +36,7 @@ describe Treat::Entities::Document do
         "a document with the contents of the file" do
           url = 'http://www.rubyinside.com/nethttp-cheat-sheet-2940.html'
           d = Treat::Entities::Document.build(url)
-          d.format.should eql :html
+          d.format.should eql 'html'
           d.print_tree
           d.should be_an_instance_of Treat::Entities::Document
           d.to_s.index('Rubyist').should_not eql nil
@@ -75,7 +75,7 @@ describe Treat::Entities::Document do
 
       context "when called on an HTML document" do
         doc = Treat::Entities::Document.new(
-        Treat.spec + 'samples/mathematicians/euler.html').read(:html)
+        Treat.paths.spec + 'samples/mathematicians/euler.html').read(:html)
         it "splits the HTML document into sections, " +
         "titles, paragraphs and lists" do
           doc.chunk
@@ -88,7 +88,7 @@ describe Treat::Entities::Document do
 
       context "when called on a text document" do
 
-        doc = Treat::Entities::Document.new(Treat.spec +
+        doc = Treat::Entities::Document.new(Treat.paths.spec +
         'samples/mathematicians/leibniz.txt').read(:txt)
         it "splits the document into titles and paragraphs" do
           doc.chunk