treat 2.0.0 → 2.0.1

data/README.md

@@ -20,6 +20,17 @@ Treat is a toolkit for natural language processing and computational linguistics
 
 I am actively seeking developers that can help maintain and expand this project. You can find a list of ideas for contributing to the project [here](https://github.com/louismullie/treat/wiki/Contributing).
 
+**Authors**
+
+Lead developper: @louismullie [[Twitter](https://twitter.com/LouisMullie)]
+
+Contributors:
+- @bdigital
+- @automatedtendencies
+- @LeFnord
+- @darkphantum
+- @whistlerbrk
+
 **License**
 
 This software is released under the [GPL License](https://github.com/louismullie/treat/wiki/License-Information) and includes software released under the GPL, Ruby, Apache 2.0 and MIT licenses.

data/lib/treat/config/data/core.rb

@@ -45,7 +45,9 @@
           spec: 'spec test files'
         }
       },
-    
+  learning: {
+    list: [:data_set, :export, :feature, :tag, :problem, :question]
+  },
   syntax: { sweetened: false },
 
   verbosity: { debug: false, silence: true}

data/lib/treat/config/data/languages/agnostic.rb

@@ -3,7 +3,7 @@
     'nokogiri', 'ferret',
     'bson_ext', 'mongo', 'lda-ruby',
     'stanford-core-nlp', 'linguistics',
-    'ruby-readability', 'whatlanguage',
+    'jruby-readability', 'whatlanguage',
     'chronic', 'nickel', 'decisiontree',
     'rb-libsvm', 'ruby-fann', 'zip',
     'tf-idf-similarity', 'narray'

data/lib/treat/core/dsl.rb

@@ -1,53 +1,21 @@
 module Treat::Core::DSL
   
-  # Message for deprecation of old DSL syntax.
-  DeprecationMessage = "The DSL that used " +
-  "capitalized entity names is now deprecated. " +
-  "Use `include Treat::Core::DSL` along with " +
-  "lowercase names from now on." 
-  
   # Map all classes in Treat::Entities to
   # a global builder function (entity, word,
   # phrase, punctuation, symbol, list, etc.)
   def self.included(base)
-    self.sweeten_entities(base)
-    self.sweeten_learning(base)
-  end
-  
-  # Map all classes in Treat::Entities to
-  # a global builder function (word, etc.)
-  def self.sweeten_entities(base, on = true)
-    Treat.core.entities.list.each do |type|
-      kname = type.cc.intern
-      mname = type.intern
-      klass = Treat::Entities.const_get(kname)
-      base.class_eval do
-        define_method(mname.capitalize) do |*args|
-          raise DeprecationMessage
-        end
-        old_mm = instance_method(:method_missing)
-        define_method(:method_missing) do |sym,*args,&block|
-          return klass.build(*args) if sym == mname
-          old_mm.bind(self).call(sym,*args,&block)
-        end
-      end
-    end
-  end
-  
-  # Map all classes in the Learning module
-  # to a global builder function. Defines:
-  # dataset, export, feature, tag, problem
-  # question.
-  def self.sweeten_learning(base, on = true)
-    Treat::Learning.constants.each do |kname|
-      mname = kname.downcase
-      klass = Treat::Learning.const_get(kname)
-      base.class_eval do
-        old_mm = instance_method(:method_missing)
-        define_method(:method_missing) do |sym,*args,&block|
-          return klass.new(*args) if sym == mname
-          old_mm.bind(self).call(sym,*args,&block)
-        end
+    def method_missing(sym,*args,&block)
+      @@entities ||= Treat.core.entities.list
+      @@learning ||= Treat.core.learning.list
+      if @@entities.include?(sym)
+        klass = Treat::Entities.const_get(sym.cc)
+        return klass.build(*args)
+      elsif @@learning.include?(sym)
+        klass = Treat::Learning.const_get(sym.cc)
+        return klass.new(*args)
+      else
+        super(sym,*args,&block)
+        raise "Uncaught method ended up in Treat DSL."
       end
     end
   end

data/lib/treat/version.rb

@@ -1,7 +1,7 @@
 module Treat
   
   # The current version of Treat.
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
   
   # Treat requires Ruby >= 1.9.2
   if RUBY_VERSION < '1.9.2'

data/lib/treat/workers/extractors/name_tag/stanford.rb

@@ -25,7 +25,7 @@ class Treat::Workers::Extractors::NameTag::Stanford
     isolated_token = entity.is_a?(Treat::Entities::Token)
     tokens = isolated_token ? [entity] : entity.tokens
 
-    ms = StanfordCoreNLP::Config::Models[:ner][language]
+    ms = StanfordCoreNLP::Config::Models[:ner][language.intern]
     model_path = Treat.libraries.stanford.model_path ||
     (Treat.paths.models + '/stanford/')
     ms = model_path + '/' + 

data/lib/treat/workers/extractors/topic_words/lda.rb

@@ -54,7 +54,7 @@ class Treat::Workers::Extractors::TopicWords::LDA
     # starting points
     
     Treat.core.verbosity.silence ?
-    silence_stdout { lda.em('random') :
+    silence_stdout { lda.em('random') }  :
     lda.em('random')
     
     # Load the vocabulary.

data/lib/treat/workers/formatters/readers/autoselect.rb

@@ -12,7 +12,9 @@ class Treat::Workers::Formatters::Readers::Autoselect
   #  - (Symbol) :default_to => format to default to.
   def self.read(document, options = {})
     options = DefaultOptions.merge(options)
-    document.read(detect_format(document.file, options[:default_to]))
+    fmt = detect_format(document.file, options[:default_to])
+    Treat::Workers::Formatters::Readers.
+    const_get(fmt.cc).read(document,options)
   end
   
   def self.detect_format(filename, default_to = nil)

data/lib/treat/workers/formatters/readers/html.rb

@@ -6,12 +6,13 @@
 # https://github.com/iterationlabs/ruby-readability
 class Treat::Workers::Formatters::Readers::HTML
   
-  silence_warnings { require 'ruby-readability' }
+  silence_warnings { require 'jruby-readability' }
 
   # By default, don't backup the original HTML
   DefaultOptions = {
     :keep_html => false,
-    :tags => %w[p div h1 h2 h3 ul ol dl dt li]
+    :tags => %w[p div h1 h2 h3 ul ol dl dt li img],
+    
   }
 
   # Read the HTML document and strip it of its markup.
@@ -46,6 +47,7 @@ class Treat::Workers::Formatters::Readers::HTML
       d = Readability::Document.new(html, options)
       document.value = "<h1>#{d.title}</h1>\n" + d.content
       document.set :format, 'html'
+      document.set :images, d.images
     end
 
     document

data/lib/treat/workers/formatters/serializers/xml.rb

@@ -17,7 +17,7 @@ class Treat::Workers::Formatters::Serializers::XML
     string += "#{val}\n</treat>"
     File.open(options[:file], 'w') do |f|
       f.write(string)
-    end; return string
+    end; return options[:file]
   end
   
   def self.recurse(entity, options)

data/lib/treat/workers/groupable.rb

@@ -69,9 +69,7 @@ module Treat::Workers::Groupable
 
   # Get constants in this module, excluding by
   # default those defined by parent modules.
-  def const_get(const)
-    super(const, false)
-  end
+  def const_get(const); super(const, false); end
 
   # Modify the extended class.
   def self.extended(group)

data/lib/treat/workers/lexicalizers/categorizers/from_tag.rb

@@ -28,8 +28,9 @@ class Treat::Workers::Lexicalizers::Categorizers::FromTag
 
     tag = entity.check_has(:tag)
     
-    return 'unknown' if tag.nil? || tag == '' || entity.type == :symbol
-    return 'sentence' if tag == 'S' || entity.type == :sentence
+    return 'unknown' if tag.nil? || tag == ''
+    return 'fragment' if tag == 'F'
+    return 'sentence' if tag == 'S'
     return 'number' if entity.type == :number
     
     return Ptc[entity.to_s] if entity.type == :punctuation

data/lib/treat/workers/lexicalizers/sensers/wordnet.rb

@@ -29,9 +29,19 @@ class Treat::Workers::Lexicalizers::Sensers::Wordnet
     
     category = word.check_has(:category)
     
-    unless options[:nym]
+    if !options[:nym] 
       raise Treat::Exception, "You must supply " +
-      "the :nym option (:synonym, :hypernym, etc.)"
+      "the :nym option ('synonyms', 'hypernyms', etc.)"
+    end
+
+    if !options[:nym].is_a?(Symbol)
+      options[:nym] == options[:nym].intern
+    end
+  
+    if ![:synonyms, :antonyms,
+      :hypernyms, :hyponyms].include?(options[:nym])
+      raise Treat::Exception, "You must supply " +
+      "a valid :nym option ('synonyms', 'hypernyms', etc.)" 
     end
     
     unless ['noun', 'adjective', 'verb'].

data/lib/treat/workers/lexicalizers/taggers/brill.rb

@@ -47,7 +47,8 @@ class Treat::Workers::Lexicalizers::Taggers::Brill
     
     return 'S' if entity.is_a?(Treat::Entities::Sentence)
     return 'P' if entity.is_a?(Treat::Entities::Phrase)
-
+    return 'F' if entity.is_a?(Treat::Entities::Fragment)
+    return 'G' if entity.is_a?(Treat::Entities::Group)
   end
 
 end

data/lib/treat/workers/lexicalizers/taggers/lingua.rb

@@ -65,9 +65,11 @@ class Treat::Workers::Lexicalizers::Taggers::Lingua
       !entity.parent_sentence
         entity.set :tag_set, :penn
     end
-
+    
     return 'S' if entity.is_a?(Treat::Entities::Sentence)
     return 'P' if entity.is_a?(Treat::Entities::Phrase)
+    return 'F' if entity.is_a?(Treat::Entities::Fragment)
+    return 'G' if entity.is_a?(Treat::Entities::Group)
 
   end
   

data/lib/treat/workers/lexicalizers/taggers/stanford.rb

@@ -32,11 +32,10 @@ class Treat::Workers::Lexicalizers::Taggers::Stanford
       entity.set :tag_set, tag_set
     end
 
-    if entity.is_a?(Treat::Entities::Sentence)
-      return 'S'
-    elsif entity.is_a?(Treat::Entities::Phrase)
-      return 'P'
-    end
+    return 'S' if entity.is_a?(Treat::Entities::Sentence)
+    return 'P' if entity.is_a?(Treat::Entities::Phrase)
+    return 'F' if entity.is_a?(Treat::Entities::Fragment)
+    return 'G' if entity.is_a?(Treat::Entities::Group)
     
     # Handle options and initialize the tagger.
     lang = entity.language

data/spec/entities/collection.rb

@@ -15,7 +15,7 @@ describe Treat::Entities::Collection do
         it "recursively searches the folder for " +
         "files and opens them into a collection of documents" do
           collection = Treat::Entities::Collection.build(@file)
-          collection.size.should eql 5
+          collection.size.should eql 6
         end
 
       end
@@ -40,7 +40,7 @@ describe Treat::Entities::Collection do
         f = Treat.paths.spec + 'workers/examples/english/economist'
         c = Treat::Entities::Collection.build(f)
         c << Treat::Entities::Document.new
-        c.size.should eql 3
+        c.size.should eql 4
       end
 
   end

data/spec/entities/entity.rb

@@ -391,11 +391,11 @@ describe Treat::Entities::Entity do
       context "when language detection is disabled " +
       "(Treat.core.detect is set to false)" do
         it "returns the default language (Treat.core.language.default)" do
-           #Treat.core.language.detect = false
-          # Treat.core.language.default = :test
+          Treat.core.language.detect = false
+          Treat.core.language.default = :test
           s = 'Les grands hommes ne sont pas toujours grands, dit un jour Napoleon.'
-          # s.language.should eql :test
-          # Treat.core.language.default = :english
+          s.language.should eql :test
+          Treat.core.language.default = :english
         end
       end
 

data/spec/helper.rb

@@ -1,13 +1,8 @@
 require_relative '../lib/treat'
+
 module Treat::Specs
 
-  # Require the worker specs.
-  require_relative 'workers'
-  # Require RSpec library.
   require 'rspec'
-  # Require Ruby benchmark library.
-  require 'benchmark'
-  # Require gem to build ASCII tables.
   
   # Some configuration options for devel.
   Treat.databases.mongo.db = 'treat_test'
@@ -20,30 +15,11 @@ module Treat::Specs
   Treat.libraries.reuters.model_path =
   '/ruby/reuters/'
   
+  ModuleFiles = ['entities/*.rb', 'learning/*.rb']
+  
   # Provide helper functions for running specs.
   class Helper
     
-    ModuleFiles = [
-      './spec/core/*.rb', 
-      './spec/entities/*.rb'
-    ]
-    
-    # Run all worker example files as :specs
-    # or :benchmarks for the given language.
-    def self.run_examples_as(what, language)
-      self.require_language_files(language)
-      Treat::Specs::Workers::Language.
-      list.each { |l| l.new(what).run }
-      RSpec::Core::CommandLine.new([]).run($stderr, $stdout)
-    end
-    
-    # Run specs for the core classes.
-    def self.run_core_specs
-      RSpec::Core::Runner.run(
-      ModuleFiles.map { |d| Dir.glob(d) }, 
-      $stderr, $stdout)
-    end
-    
     # Start SimpleCov coverage.
     def self.start_coverage
       require 'simplecov'
@@ -61,56 +37,28 @@ module Treat::Specs
       end
     end
     
+    # Run specs for the core classes.
+    def self.run_core_specs
+      files = ModuleFiles.map do |d|
+        Dir.glob(Treat.paths.spec + d)
+      end
+      RSpec::Core::Runner.run(files)
+    end
+    
     # Require language files based on the argument.
-    def self.require_language_files(arg)
-      # Require the base language class.
-      require_relative 'workers/language'
+    def self.run_language_specs(lang)
       # If no language supplied, get all languages.
-      if !arg || arg == ''
+      if !lang || lang == ''
         pattern = "./spec/workers/*.rb"
       # Otherwise, get a specific language file.
       else
-        pattern = "./spec/workers/#{arg}.rb"
-        # Check if a spec file exists.
+        pattern = "./spec/workers/#{lang}.rb"
         unless File.readable?(pattern)
           raise Treat::Exception, 
-          "There are no examples for '#{arg}'."
-        end
-      end
-      # Require all files matched by the pattern.
-      Dir.glob(pattern).each { |f| require f }
-    end
-    
-    def self.text_table(headings, rows)
-      require 'terminal-table'
-      puts Terminal::Table.new(
-      headings: headings, rows: rows)
-    end
-
-    def self.html_table(headings, rows)
-      require 'fileutils'
-      html = "<table>\n"
-      html += "<tr>\n"
-      headings.each do |heading|
-        html += "<td>" + heading + "</td>\n"
-      end
-      html += "</tr>\n"
-      rows.each do |row|
-        html += "<tr>\n"
-        row.each do |el|
-          html += "<td>#{el}</td>"
+          "There are no examples for '#{lang}'."
         end
-        html += "</tr>\n"
-      end
-      self.write_html('benchmark', html)
-    end
-    
-    def self.write_html(dir, html)
-      unless FileTest.directory?(dir) 
-        FileUtils.mkdir('./' + dir) 
       end
-      fn = "./#{dir}/index.html"
-      File.open(fn, 'w+') { |f| f.write(html) }
+      RSpec::Core::Runner.run(Dir.glob(pattern))
     end
     
   end

data/spec/sandbox.rb

@@ -4,6 +4,15 @@ require_relative '../lib/treat'
 require 'treat'
 include Treat::Core::DSL
 
+collection Treat.paths.spec + '/workers/examples/english/economist'
+collection.apply :chunk, :segment, :tokenize
+puts collection.topic_words.inspect
+
+=begin
+
+g = group("I was running")
+puts g.tag.inspect
+
 Treat.libraries.stanford.jar_path = '/ruby/treat/bin/'
 Treat.libraries.stanford.model_path = '/ruby/treat/models/'
 
@@ -14,6 +23,8 @@ w = word
 p = phrase 'hello world'
 e = email 'louis@gmail.com'
 
+d = question(:is_feature, :word)
+=end
 #d = document Treat.paths.spec + 'workers/examples/english/economist/hungarys_troubles.txt'
 #d.apply :chunk, :segment, :tokenize, :tag, :category, :name_tag
 #d.print_tree
@@ -266,6 +277,6 @@ sect = section title(phra), para
 =begin
 puts "beer".plural.inspect
 =end
-Treat.core.language.detect = true
-s = sentence "Du hast deiner Frau einen roten Ring gekauft."
-s.apply(:parse,:category).print_tree
+# Treat.core.language.detect = true
+# s = sentence "Du hast deiner Frau einen roten Ring gekauft."
+#s.apply(:parse,:category).print_tree

data/spec/workers/agnostic.rb

@@ -1,3 +1,80 @@
+$workers = Treat.languages.agnostic.workers
+
+describe Treat::Workers::Extractors::Language do
+  before do
+    @entities = ["Obama and Sarkozy will meet in Berlin."]
+    @languages = ["english"]
+  end
+  context "when called on any textual entity" do
+    it "returns the language of the entity" do
+      # Treat.core.language.detect = true
+      $workers.extractors.language.each do |extractor|
+        @entities.map(&:language).should eql @languages
+      end
+      # Treat.core.language.detect = false
+    end
+  end
+end
+
+describe Treat::Workers::Formatters::Serializers do
+  before do
+    @texts = ["A test entity"]
+  end
+  context "when #serialize is called on any textual entity" do
+    it "serializes the entity to disk and returns a pointer to the location" do
+      # m = Treat::Entities::Entity.build
+      @texts.map(&:to_entity).map(&:serialize)
+      .map(&method(:entity)).map(&:to_s).should eql @texts
+    end
+  end
+end
+
+describe Treat::Workers::Formatters::Unserializers do
+  before do
+    @texts = ["A te"]
+  end
+  context "when #unserialize is called with a selector on any textual entity" do
+    it "unserializes the file and loads it in the entity" do
+      
+    end
+  end
+end
+
+visualize: {
+  entity: {
+    examples: {
+      standoff: [
+        ["I walked to the store.", "(S\n   (PRP I)   (VBD walked)   (TO to)   (DT the)   (NN store)   (. .))\n"]
+      ],
+      tree: [
+        ["I walked to the store.", "+ Sentence (*)  --- \"I walked to the store.\"  ---  {}   --- [] \n|\n+--> Word (*)  --- \"I\"  ---  {}   --- [] \n+--> Word (*)  --- \"walked\"  ---  {}   --- [] \n+--> Word (*)  --- \"to\"  ---  {}   --- [] \n+--> Word (*)  --- \"the\"  ---  {}   --- [] \n+--> Word (*)  --- \"store\"  ---  {}   --- [] \n+--> Punctuation (*)  --- \".\"  ---  {}   --- [] "]
+      ],
+      dot: [
+        ["I walked to the store.", "graph {\n* [label=\"Sentence\\n\\\"I walked to the store.\\\"\",color=\"\"]\n* [label=\"Word\\n\\\"I\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"walked\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"to\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"the\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"store\\\"\",color=\"\"]\n* -- *;\n* [label=\"Punctuation\\n\\\".\\\"\",color=\"\"]\n* -- *;\n}"]
+      ]
+    },
+    preprocessor: lambda  { |entity| entity.tokenize },
+    generator: lambda  { |result| result.gsub(/[0-9]+/, '*') }
+  }
+},
+
+
+describe Treat::Workers::Formatters::Visualizers do
+  before do
+    @texts = ["I walked to the store."]
+  end
+  describe "when #visualize is called with the :dot worker" do
+    
+  end
+  describe "when #visualize is called with the :tree worker" do
+    
+  end
+  describe "when #visualize is called with the :dot worker" do
+    
+  end
+end
+
+=begin
 class Treat::Specs::Workers::Agnostic < Treat::Specs::Workers::Language
 
   # TODO: :tf_idf, :keywords, :classifiers
@@ -5,15 +82,6 @@ class Treat::Specs::Workers::Agnostic < Treat::Specs::Workers::Language
 
   Scenarios = {
 
-    # Also tests unserialize.
-    serialize: {
-      entity: {
-        examples: [
-          ["A test entity.", "A test entity."]
-        ],
-        generator: lambda { |selector| Treat::Entities::Entity.build(selector).to_s }
-      }
-    },
     classify: {
       entity: {
         examples: [
@@ -39,23 +107,6 @@ class Treat::Specs::Workers::Agnostic < Treat::Specs::Workers::Language
         end
       }
     },
-    visualize: {
-      entity: {
-        examples: {
-          standoff: [
-            ["I walked to the store.", "(S\n   (PRP I)   (VBD walked)   (TO to)   (DT the)   (NN store)   (. .))\n"]
-          ],
-          tree: [
-            ["I walked to the store.", "+ Sentence (*)  --- \"I walked to the store.\"  ---  {}   --- [] \n|\n+--> Word (*)  --- \"I\"  ---  {}   --- [] \n+--> Word (*)  --- \"walked\"  ---  {}   --- [] \n+--> Word (*)  --- \"to\"  ---  {}   --- [] \n+--> Word (*)  --- \"the\"  ---  {}   --- [] \n+--> Word (*)  --- \"store\"  ---  {}   --- [] \n+--> Punctuation (*)  --- \".\"  ---  {}   --- [] "]
-          ],
-          dot: [
-            ["I walked to the store.", "graph {\n* [label=\"Sentence\\n\\\"I walked to the store.\\\"\",color=\"\"]\n* [label=\"Word\\n\\\"I\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"walked\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"to\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"the\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"store\\\"\",color=\"\"]\n* -- *;\n* [label=\"Punctuation\\n\\\".\\\"\",color=\"\"]\n* -- *;\n}"]
-          ]
-        },
-        preprocessor: lambda  { |entity| entity.tokenize },
-        generator: lambda  { |result| result.gsub(/[0-9]+/, '*') }
-      }
-    },
 
 =begin
     keywords: {
@@ -103,8 +154,7 @@ class Treat::Specs::Workers::Agnostic < Treat::Specs::Workers::Language
           preprocessor: lambda { |coll| coll.apply(:index) }
       },
     },
-=end
-=begin
+
     keywords: {
       document: {
         examples: [
@@ -124,7 +174,7 @@ class Treat::Specs::Workers::Agnostic < Treat::Specs::Workers::Language
         ]
       }
     },
-=end
+
     topic_words: {
       collection: {
         examples: [
@@ -134,4 +184,4 @@ class Treat::Specs::Workers::Agnostic < Treat::Specs::Workers::Language
     }
   }
 
-end
+=end