treat 2.0.2 → 2.0.3

data/lib/treat/workers/categorizable.rb

@@ -9,26 +9,24 @@ module Treat::Workers::Categorizable
   @@lookup = {}
 
   # Find a worker group based on method.
-  def lookup(method)
-    @@lookup[method]
-  end
+  def lookup(method); @@lookup[method]; end
 
   def categorize!
     Treat.workers.members.each do |cat|
-      create_category(cat.
-      capitalize.intern,
-      load_category_conf(cat))
+      name = cat.capitalize.intern
+      conf = load_category_conf(cat)
+      create_category(name, conf)
     end
   end
 
   def load_category_conf(name)
-    config = Treat.workers[name]
-    if config.nil?
+    if !Treat.workers.respond_to?(name)
       raise Treat::Exception,
       "The configuration file " +
       "for #{cat_sym} is missing."
+    else
+      Treat.workers[name]
     end
-    config
   end
 
   def create_category(name, conf)
@@ -37,11 +35,11 @@ module Treat::Workers::Categorizable
     conf.each_pair do |group, worker|
       name = group.to_s.cc.intern
       category.module_eval do
-        @@methods = []; def methods; 
-        @@methods; end; def groups; 
-        self.constants; end
+        @@methods = []
+        def methods; @@methods; end
+        def groups; self.constants; end
       end
-      self.create_group(name, worker, category)
+      create_group(name, worker, category)
     end
   end
 
@@ -54,24 +52,6 @@ module Treat::Workers::Categorizable
     @@lookup[group.method] = group
   end
 
-  def bind_group_targets(group)
-    group.targets.each do |entity_type|
-      entity = Treat::Entities.
-      const_get(entity_type.cc)
-      entity.class_eval do
-        add_workers group
-      end
-    end
-  end
-
-  def register_group_presets(group, conf)
-    return unless conf.respond_to? :presets
-    conf.presets.each do |m|
-      @@methods << m
-      @@lookup[m] = group
-    end
-  end
-
   def set_group_options(group, conf)
     group.module_eval do
       extend Treat::Workers::Groupable
@@ -91,5 +71,23 @@ module Treat::Workers::Categorizable
       end
     end
   end
-  
+
+  def bind_group_targets(group)
+    group.targets.each do |entity_type|
+      entity = Treat::Entities.
+      const_get(entity_type.cc)
+      entity.class_eval do
+        add_workers group
+      end
+    end
+  end
+
+  def register_group_presets(group, conf)
+    return unless conf.respond_to?(:presets)
+    conf.presets.each do |method|
+      @@methods << method
+      @@lookup[method] = group
+    end
+  end
+
 end

data/lib/treat/workers/extractors/name_tag/stanford.rb

@@ -16,32 +16,21 @@ class Treat::Workers::Extractors::NameTag::Stanford
 
   def self.name_tag(entity, options = {})
 
-    pp = nil
-
     language = entity.language
-  
     Treat::Loaders::Stanford.load(language)
     
     isolated_token = entity.is_a?(Treat::Entities::Token)
     tokens = isolated_token ? [entity] : entity.tokens
-
-    ms = StanfordCoreNLP::Config::Models[:ner][language.intern]
-    model_path = Treat.libraries.stanford.model_path ||
-    (Treat.paths.models + '/stanford/')
-    ms = model_path + '/' + 
-    StanfordCoreNLP::Config::ModelFolders[:ner] +
-    ms['3class']
-
-    @@classifiers[language] ||=
-    StanfordCoreNLP::CRFClassifier.
-    getClassifier(ms)
-
+    
+    unless classifier = @@classifiers[language]
+      model = Treat::Loaders::Stanford.find_model(:ner, language)
+      classifier = StanfordCoreNLP::CRFClassifier.getClassifier(model)
+      @@classifiers[language] = classifier
+    end
+    
     token_list = StanfordCoreNLP.get_list(tokens)
-    sentence = @@classifiers[language].
-    classify_sentence(token_list)
-
+    sentence = classifier.classify_sentence(token_list)
     i = 0
-    n = 0
     
     sentence.each do |s_token|
       tag = s_token.get(:answer).to_s.downcase
@@ -49,14 +38,9 @@ class Treat::Workers::Extractors::NameTag::Stanford
       return tag if isolated_token
       if tag
         tokens[i].set :name_tag, tag
-        n += 1
       end
       i += 1
     end
-
-    entity.set :named_entity_count, n
-
-    nil
     
   end
 

data/lib/treat/workers/formatters/readers/html.rb

@@ -6,7 +6,7 @@
 # https://github.com/iterationlabs/ruby-readability
 class Treat::Workers::Formatters::Readers::HTML
   
-  silence_warnings { require 'jruby-readability' }
+  silence_warnings { require 'ruby-readability' }
 
   # By default, don't backup the original HTML
   DefaultOptions = {

data/lib/treat/workers/formatters/readers/xml.rb

@@ -30,7 +30,7 @@ class Treat::Workers::Formatters::Readers::XML
     @@xml_reader ||= StanfordCoreNLP.load(
     :tokenize, :ssplit, :cleanxml)
 
-    text = StanfordCoreNLP::Text.new(xml)
+    text = StanfordCoreNLP::Annotation.new(xml)
     @@xml_reader.annotate(text)
 
     text.get(:sentences).each do |sentence|

data/lib/treat/workers/formatters/unserializers/mongo.rb

@@ -17,7 +17,7 @@ class Treat::Workers::Formatters::Unserializers::Mongo
     
     @@database ||= Mongo::Connection.
     new(Treat.databases.mongo.host).
-    db(Treat.databases.mongo.db || db)
+    db(db || Treat.databases.mongo.db)
     
     supertype =  Treat::Entities.const_get(
     entity.type.to_s.capitalize.intern).superclass.mn.downcase

data/lib/treat/workers/groupable.rb

@@ -5,7 +5,8 @@ module Treat::Workers::Groupable
     bits = self.ancestors[0].to_s.split('::')
     bits.collect! { |bit| bit.ucc }
     file = bits.join('/') + "/#{const.ucc}"
-    if not File.readable?(Treat.paths.lib + "#{file}.rb")
+    path = Treat.paths.lib + "#{file}.rb"
+    if not File.readable?(path)
       raise Treat::Exception,
       "File '#{file}.rb' corresponding to " +
       "requested worker #{self}::#{const} " +

data/lib/treat/workers/inflectors/cardinalizers/linguistics.rb

@@ -35,9 +35,9 @@ class Treat::Workers::Inflectors::Cardinalizers::Linguistics
   # More specific options when using :type => :ordinal:
   def self.cardinal(entity, options = {})
     options = DefaultOptions.merge(options)
-    Treat::Loaders::Linguistics.
-    load(options[:language]).
-    numwords(entity.to_s, options)
+    lang = entity.language
+    code = Treat::Loaders::Linguistics.load(lang)
+    entity.to_s.send(code).numwords(options)
   end
   
 end

data/lib/treat/workers/inflectors/conjugators/linguistics.rb

@@ -35,13 +35,15 @@ module Treat::Workers::Inflectors::Conjugators::Linguistics
 
     options = Forms[options[:form].to_s] if options[:form]
 
-    klass = Treat::Loaders::Linguistics.load(entity.language)
+    code = Treat::Loaders::Linguistics.load(entity.language)
+    obj = entity.to_s.send(code)
+  
     if options[:mode] == 'infinitive'
-      silence_warnings { klass.infinitive(entity.to_s) }
+      obj.infinitive
     elsif options[:mode] == 'participle' && options[:tense] == 'present'
-      silence_warnings { klass.present_participle(entity.to_s) }
+      obj.present_participle
     elsif options[:count] == 'plural' && options.size == 1
-      silence_warnings { klass.plural_verb(entity.to_s) }
+      obj.plural_verb
     else
       raise Treat::Exception,
       'This combination of modes, tenses, persons ' +

data/lib/treat/workers/inflectors/declensors/linguistics.rb

@@ -17,34 +17,27 @@ class Treat::Workers::Inflectors::Declensors::Linguistics
 
     cat = entity.get(:category)
     return if cat && !POS.include?(cat)
+    
     unless options[:count]
       raise Treat::Exception, 'Must supply ' +
       ':count option ("singular" or "plural").'
     end
-    
-    klass = Treat::Loaders::
-    Linguistics.load(entity.language)
-    string = entity.to_s
-
-    if options[:count].to_s == 'plural'
-      if (entity.has?(:category))
-        result = ''
-        silence_warnings do
-          result = klass.send(
-          :"plural_#{entity.category}",
-          string)
-        end
-        return result
-      else
-        return klass.plural(string)
-      end
 
-    else
+    unless options[:count].to_s == 'plural'
       raise Treat::Exception,
       "Ruby Linguistics does not support " +
       "singularization of words."
     end
 
+    lang = entity.language
+    code = Treat::Loaders::Linguistics.load(lang)
+    obj = entity.to_s.send(code)
+
+    if cat = entity.get(:category)
+      method = "plural_#{cat}"
+      obj.send(method)
+    else; obj.plural; end
+
   end
 
 end

data/lib/treat/workers/inflectors/ordinalizers/linguistics.rb

@@ -11,11 +11,11 @@ class Treat::Workers::Inflectors::Ordinalizers::Linguistics
   
   # Desribe a number in words in ordinal form, using the
   # 'linguistics' gem.
-  def self.ordinal(number, options = {})
+  def self.ordinal(entity, options = {})
     options = DefaultOptions.merge(options)
-    klass = Treat::Loaders::
-    Linguistics.load(options[:language])
-    klass.ordinate(number.to_s)
+    lang = entity.language
+    code = Treat::Loaders::Linguistics.load(lang)
+    entity.to_s.send(code).ordinate
   end
   
 end

data/lib/treat/workers/lexicalizers/sensers/wordnet.rb

@@ -35,7 +35,7 @@ class Treat::Workers::Lexicalizers::Sensers::Wordnet
     end
 
     if !options[:nym].is_a?(Symbol)
-      options[:nym] == options[:nym].intern
+      options[:nym] = options[:nym].intern
     end
   
     if ![:synonyms, :antonyms,

data/lib/treat/workers/lexicalizers/taggers/stanford.rb

@@ -1,15 +1,15 @@
-# POS tagging using (i) explicit use of both preceding 
-# and following tag contexts via a dependency network 
-# representation, (ii) broad use of lexical features, 
-# including jointly conditioning on multiple consecutive 
-# words, (iii) effective use of priors in conditional 
-# loglinear models, and (iv) fine-grained modeling of 
+# POS tagging using (i) explicit use of both preceding
+# and following tag contexts via a dependency network
+# representation, (ii) broad use of lexical features,
+# including jointly conditioning on multiple consecutive
+# words, (iii) effective use of priors in conditional
+# loglinear models, and (iv) fine-grained modeling of
 # unknown word features.
 #
 # Original paper: Toutanova, Manning, Klein and Singer.
-# 2003. Feature-Rich Part-of-Speech Tagging with a 
-# Cyclic Dependency Network. In Proceedings of the 
-# Conference of the North American Chapter of the 
+# 2003. Feature-Rich Part-of-Speech Tagging with a
+# Cyclic Dependency Network. In Proceedings of the
+# Conference of the North American Chapter of the
 # Association for Computational Linguistics.
 class Treat::Workers::Lexicalizers::Taggers::Stanford
 
@@ -21,6 +21,9 @@ class Treat::Workers::Lexicalizers::Taggers::Stanford
     :tagger_model => nil
   }
 
+  # Shortcut for gem config.
+  Config = StanfordCoreNLP::Config
+
   # Tag the word using one of the Stanford taggers.
   def self.tag(entity, options = {})
 
@@ -36,9 +39,9 @@ class Treat::Workers::Lexicalizers::Taggers::Stanford
     return 'P' if entity.is_a?(Treat::Entities::Phrase)
     return 'F' if entity.is_a?(Treat::Entities::Fragment)
     return 'G' if entity.is_a?(Treat::Entities::Group)
-    
+
     # Handle options and initialize the tagger.
-    lang = entity.language
+    lang = entity.language.intern
     init_tagger(lang) unless @@taggers[lang]
     options = get_options(options, lang)
     tokens, t_list = get_token_list(entity)
@@ -46,7 +49,7 @@ class Treat::Workers::Lexicalizers::Taggers::Stanford
     # Do the tagging.
     i = 0
     isolated_token = entity.is_a?(Treat::Entities::Token)
-    
+
     @@taggers[lang].apply(t_list).each do |tok|
       tokens[i].set(:tag, tok.tag)
       tokens[i].set(:tag_set,
@@ -59,21 +62,20 @@ class Treat::Workers::Lexicalizers::Taggers::Stanford
 
   # Initialize the tagger for a language.
   def self.init_tagger(language)
-    Treat::Loaders::Stanford.load(language)
-    model = StanfordCoreNLP::Config::Models[:pos][language]
-    model_path = Treat.libraries.stanford.model_path || 
-    Treat.paths.models + 'stanford/'
-    model = model_path + StanfordCoreNLP::
-    Config::ModelFolders[:pos] + model
-    @@taggers[language] ||=
-    StanfordCoreNLP::MaxentTagger.new(model)
+    unless @@taggers[language]
+      Treat::Loaders::Stanford.load(language)
+      model = Treat::Loaders::Stanford.find_model(:pos,language)
+      tagger = StanfordCoreNLP::MaxentTagger.new(model)
+      @@taggers[language] = tagger
+    end
+    @@taggers[language]
   end
 
   # Handle the options for the tagger.
   def self.get_options(options, language)
     options = DefaultOptions.merge(options)
     if options[:tagger_model]
-      ::StanfordCoreNLP.set_model('pos.model',
+      StanfordCoreNLP.set_model('pos.model',
       options[:tagger_model])
     end
     options[:tag_set] =

data/lib/treat/workers/processors/parsers/stanford.rb

@@ -29,15 +29,14 @@ class Treat::Workers::Processors::Parsers::Stanford
   # instead of displaying it.
   def self.parse(entity, options = {})
 
-    val = entity.to_s
-    lang = entity.language
-    init(lang, options)
+    val, lang = entity.to_s, entity.language
+    init(lang, options) unless @@parsers[lang]
     
     entity.check_hasnt_children
     
     tag_set = StanfordCoreNLP::Config::TagSets[lang]
     
-    text = ::StanfordCoreNLP::Text.new(val)
+    text = ::StanfordCoreNLP::Annotation.new(val)
     @@parsers[lang].annotate(text)
 
     text.get(:sentences).each do |s|
@@ -50,7 +49,7 @@ class Treat::Workers::Processors::Parsers::Stanford
         entity.set :tag, tag_s
         entity.set :tag_opt, tag_opt if tag_opt
         recurse(s.get(:tree).children[0], entity, tag_set)
-        break #######
+        break ####### ? FIX
       else
         recurse(s.get(:tree), entity, tag_set)
       end
@@ -62,26 +61,17 @@ class Treat::Workers::Processors::Parsers::Stanford
   end
 
   def self.init(lang, options)
-    return if @@parsers[lang]
-    
     Treat::Loaders::Stanford.load(lang)
-    
     options = DefaultOptions.merge(options)
-    StanfordCoreNLP.use(lang)
+    StanfordCoreNLP.use(lang.intern)
     if options[:tagger_model]
-      ::StanfordCoreNLP.set_model(
-      'pos.model', options[:tagger_model]
-      )
+      StanfordCoreNLP.set_model('pos.model', options[:tagger_model])
     end
     if options[:parser_model]
-      ::StanfordCoreNLP.set_model(
-      'parser.model', options[:parser_model]
-      )
+      StanfordCoreNLP.set_model('parser.model', options[:parser_model])
     end
-    @@parsers[lang] ||=
-    ::StanfordCoreNLP.load(
-    :tokenize, :ssplit, :pos, :lemma, :parse
-    )
+    annotators = [:tokenize, :ssplit, :pos, :lemma, :parse]
+    @@parsers[lang] = StanfordCoreNLP.load(*annotators)
   end
 
   # Helper method which recurses the tree supplied by
@@ -128,7 +118,7 @@ class Treat::Workers::Processors::Parsers::Stanford
           l = java_child.children[0].to_s
           v = java_child.children[0].value.to_s.strip
           
-          # Mhmhmhmhmhm
+          # Mhmhmhmhmhm FIX!
           val = (l == v) ? v :  l.split(' ')[-1].gsub(')', '')
           ruby_child = Treat::Entities::Token.from_string(val)
         end

data/lib/treat/workers/processors/segmenters/stanford.rb

@@ -32,7 +32,7 @@ class Treat::Workers::Processors::Segmenters::Stanford
     ::StanfordCoreNLP.load(:tokenize, :ssplit)
     
     s = entity.to_s
-    text = ::StanfordCoreNLP::Text.new(s)
+    text = ::StanfordCoreNLP::Annotation.new(s)
 
     @@segmenter.annotate(text)
     text.get(:sentences).each do |sentence|

data/lib/treat/workers/processors/tokenizers/maxent.rb

@@ -0,0 +1,29 @@
+# Maximum entropy tokenization supplied by OpenNLP.
+class Treat::Workers::Processors::Tokenizers::Maxent
+
+  require 'open-nlp'
+  OpenNLP.load
+  
+  # Maximum entropy tokenization.
+  def self.tokenize(entity, options = {})
+    
+    lang = entity.language
+    str = entity.to_s
+    
+    unless @@tokenizers[lang]
+      OpenNLP.use(lang.intern)
+      @@tokenizers[lang] = 
+      OpenNLP::TokenizerME.new
+    end
+    
+    tokenizer = @@tokenizers[lang]
+    tokens = tokenizer.tokenize(str).to_a
+    
+    tokens.each do |token|
+      entity << Treat::Entities
+      ::Token.from_string(chunk)
+    end
+
+  end
+
+end

data/lib/treat/workers/processors/tokenizers/stanford.rb

@@ -27,11 +27,9 @@ class Treat::Workers::Processors::Tokenizers::Stanford
   # replacing "..." by ``...''. Off by default.
   def self.tokenize(entity, options = {})
     options = DefaultOptions.merge(options)
-    @@tokenizer ||=
-    ::StanfordCoreNLP.load(:tokenize)
+    @@tokenizer ||= StanfordCoreNLP.load(:tokenize)
     entity.check_hasnt_children
-    text = ::StanfordCoreNLP::
-    Text.new(entity.to_s)
+    text = ::StanfordCoreNLP::Annotation.new(entity.to_s)
     @@tokenizer.annotate(text)
     add_tokens(entity, text.get(:tokens), options)
   end

data/lib/treat.rb

@@ -18,5 +18,6 @@ module Treat
   require_relative 'treat/exception'
   require_relative 'treat/autoload'
   require_relative 'treat/modules'
+  require_relative 'treat/builder'
 
 end

data/spec/helper.rb

@@ -1,20 +1,23 @@
 require_relative '../lib/treat'
 
+include Treat::Core::DSL
+  
 module Treat::Specs
 
   require 'rspec'
   
   # Some configuration options for devel.
+
   Treat.databases.mongo.db = 'treat_test'
   Treat.libraries.stanford.model_path =
-  '/ruby/stanford/stanford-core-nlp-all/'
+  '/ruby/stanford-core-nlp-minimal/models/'
   Treat.libraries.stanford.jar_path =
-  '/ruby/stanford/stanford-core-nlp-all/'
+  '/ruby/stanford-core-nlp-minimal/bin/'
   Treat.libraries.punkt.model_path =
-  '/ruby/punkt/'
+  '/ruby/punkt/models/'
   Treat.libraries.reuters.model_path =
-  '/ruby/reuters/'
-  
+  '/ruby/reuters/models/'
+
   ModuleFiles = ['entities/*.rb', 'learning/*.rb']
   
   # Provide helper functions for running specs.
@@ -25,7 +28,6 @@ module Treat::Specs
       require 'simplecov'
       SimpleCov.start do
         add_filter '/spec/'
-        add_filter '/config/'
         add_group 'Core', 'treat/core'
         add_group 'Entities', 'treat/entities'
         add_group 'Helpers', 'treat/helpers'

data/spec/sandbox.rb

@@ -1,13 +1,25 @@
 # encoding: utf-8
 require_relative '../lib/treat'
 
-require 'treat'
-include Treat::Core::DSL
-
-collection Treat.paths.spec + '/workers/examples/english/economist'
-collection.apply :chunk, :segment, :tokenize
-puts collection.topic_words.inspect
+Treat.databases.mongo.db = 'treat_test'
+Treat.libraries.stanford.model_path =
+'/ruby/stanford-core-nlp-minimal/models/'
+Treat.libraries.stanford.jar_path =
+'/ruby/stanford-core-nlp-minimal/bin/'
+Treat.libraries.punkt.model_path =
+'/ruby/punkt/models/'
+Treat.libraries.reuters.model_path =
+'/ruby/reuters/models/'
+
+# include Treat::Core::DSL
+
+Treat::Builder.new do
+  s = sentence "Hello, world!"
+  s.print_tree
+end
 
+p = paragraph('A walk in the park. A trip on a boat.').segment
+p.visualize :dot, file: 'test.dot'
 =begin
 
 g = group("I was running")