treat 1.2.0 → 2.0.0rc1

data/spec/sandbox.rb

@@ -1,36 +1,269 @@
-require_relative 'helper'
+# encoding: utf-8
+require_relative '../lib/treat'
+
+require 'treat'
+include Treat::Core::DSL
+
+Treat.libraries.stanford.jar_path = '/ruby/treat/bin/'
+Treat.libraries.stanford.model_path = '/ruby/treat/models/'
+
+p = paragraph
+s = sentence
+w = word
+
+p = phrase 'hello world'
+e = email 'louis@gmail.com'
+
+#d = document Treat.paths.spec + 'workers/examples/english/economist/hungarys_troubles.txt'
+#d.apply :chunk, :segment, :tokenize, :tag, :category, :name_tag
+#d.print_tree
+#d = document Treat.paths.spec + 'workers/examples/english/economist/saving_the_euro.odt'
+#d.print_tree
+=begin
+d = document 'test.htm'
+d.apply :chunk
+#d.serialize :yaml, file: 'test444.yaml'
+d.set :test, 2
+d.serialize :mongo, db: 'test'
+d.set :test, 3
+d.serialize :mongo, db: 'test'
+d.apply :segment, :tokenize, :tag, :category
+puts d.verb_count
+#d2 = document id: d.id, db: 'test'
+d2 = document 'features.test' => 3, db: 'test'
+d2.apply :segment, :tokenize, :tag, :category
+puts d2.verb_count
+#d.print_tree
+#s = document 'http://www.economist.com'
+
+p = phrase 'hello', 'world', '!'
+puts p.to_s 
+puts p.to_str
+=end
+
+=begin
+### Super basics.
+puts p.value
+
+p << 'bitch'
+p << word('hello')
+puts p.to_s 
+puts p.to_str
+puts p.value
+puts p.to_ary.inspect
+=end
+
+=begin
+
+### Configuration
+
+# A boolean value indicating whether to silence the output of external libraries (e.g. Stanford tools, Enju, LDA, Ruby-FANN) when they are used.
+puts Treat.core.verbosity.silence
+# A boolean value indicating whether to explain the steps that Treat is performing.
+puts Treat.core.verbosity.debug
+# A boolean value indicating whether Treat should try to detect the language of newly input text.
+puts Treat.core.language.detect
+# The language to default to when detection is off.
+puts Treat.core.language.default
+# A symbol representing the finest level at which language detection should be performed if language detection is turned on.
+puts Treat.core.language.detect_at
+
+# A directory in which to create temporary files.
+puts Treat.paths.tmp
+# A directory in which to store downloaded files.
+puts Treat.paths.files
+# A directory containing trained models for various tasks.
+puts Treat.paths.models
+# A directory containing the spec files.
+puts Treat.paths.spec
+# A directory containing executables and JAR files.
+puts Treat.paths.bin
+puts Treat.paths.lib
+
+# Set up Mongoid.
+Treat.databases.mongo.db = 'your_database'
+Treat.databases.mongo.host = 'localhost'
+Treat.databases.mongo.port = '27017'
+
+# Transparent string casting.
+s = 'inflection'.stem
+# is equivalent to
+s = 'inflection'.to_entity.stem
+# which comes down to
+s = word('inflection').stem
+
+# Transparent number casting.
+n = 2.ordinal
+# is equivalent to
+s = 2.to_entity.ordinal
+# which comes down to
+s = number(2).ordinal
+=end
 =begin
-problem = Problem(
-  Question(:is_key_sentence, :sentence, :discrete, 0, [0, 1]),
-  Feature(:word_count, 0),
-  Tag(:number_count, 0)
-)
+### BASIC USAGE
 
-problem.id = 70316753228720
+# Create a sentence
+s = sentence 'Those who dream by day know of at least ' +
+'19 things that escape those who dream only at night.'
 
-test = Paragraph("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged.")
+# Tokenize and tag it.
+s.tokenize.tag
 
-test.do :segment, :tokenize, :tag, :category
+# View the sentence structure.
+s.print_tree
 
-test.sentences[0].set :is_key_sentence, 1
-test.sentences[1].set :is_key_sentence, 1
-test.sentences[2].set :is_key_sentence, 0
+# Iterate over the tokens.
+s.each_token do |tok|
+  puts tok.value
+  puts tok.type
+end
 
-ds = test.export(problem)
 
-test.each_sentence do |s|
-  puts s.classify :linear, training: ds
+
+# Arrays instead of iterators.
+(s.nouns + s.adjectives).each do |word|
+  puts word.synonyms
+  puts word.antonyms
+end
+
+# Functions on numbers.
+s.each_number do |num|
+  puts num.ordinal
+  puts num.cardinal
 end
+
+# See all the annotations.
+s.each do |tok|
+  puts tok.inspect
+end
+
+# Lazy way of doing all of the above.
+s = sentence 'Those who dream by day know of at least ' +
+'19 things that escape those who dream only at night.'
+
+s.apply :tokenize, :tag, :category,
+        :stem, :hyponyms, :hypernyms,
+        :antonyms, :ordinal, :cardinal
+
 =end
+
 =begin
-Treat.databases.mongo.db = 'testing_ds'
-ds1 = Treat::Core::DataSet.unserialize :marshal, file: 'test.dump'
-ds1.serialize :mongo
-puts ds1.problem.id
-ds = Treat::Core::DataSet.unserialize :mongo, {problem: ds1.problem.id}
-puts ds.inspect
+### A BIT MORE ADVANCED USAGE
+
+section = section "Obama-Sarkozy Meeting\n" +
+"Obama and Sarkozy met on January 1st to investigate " +
+"the possibility of a new rescue plan. President " +
+"Sarkozy is to meet Merkel next Tuesday in Berlin."
+
+# Chunk: split the titles and paragraphs.
+# Segment: perform sentence segmentation.
+# Parse: parse the syntax of each sentence.
+section.apply :chunk, :segment, :parse
+
+# View the tree structure.
+section.print_tree
 
-Treat.databases.mongo.db = 'testingyetagain'
-ds = DataSet(:dump, './all.dump')
-ds.to_mongo({db: 'test_ds', collection: 'data'})
-=end
+# Get some basic info on the text.
+puts section.title
+puts section.sentence_count
+puts section.word_count
+
+section.apply :category
+puts section.noun_count
+puts section.frequency_of 'president'
+
+section.each_phrase_with_tag('NP') do |phrase|
+  puts phrase.to_s
+end
+
+=end
+=begin
+### URL documents, XML serialization.
+
+urls = ['http://www.cbc.ca/news/world/story/2012/11/25/snc-lavalin-ben-aissa-charges.html',
+'http://www.cbc.ca/news/world/story/2012/11/25/egypt.html', 'http://www.cbc.ca/news/canada/prince-edward-island/story/2012/11/25/pei-murder-arrest-stlucia.html', 'http://www.cbc.ca/news/world/story/2012/11/25/bangladesh-garment-factory-fire.html']
+
+c = collection
+urls.each { |url| c << document(url) }
+
+# View the collection.
+c.print_tree
+
+c.apply :chunk, :segment, :tokenize
+c.serialize :xml, :file => 'test.xml'
+
+# Reopen the collection.
+c = collection 'test.xml'
+
+# View it again.
+c.print_tree
+=end
+=begin
+include Treat::Core::DSL
+
+# Show progress bars for download.
+Treat.core.verbosity.silence = false
+# Explain what Treat is doing.
+Treat.core.verbosity.debug = true
+
+# Define the question "is it junk?" on sentences.
+qn = question(:is_junk, :sentence)
+
+# Frame the problem as depending on punctuation
+# count and word count for each sentence.
+pb = problem(qn,
+     feature(:punctuation_count), 
+     feature(:word_count) )
+
+# Get some web documents to work on.
+url1 = 'http://en.wikipedia.org/wiki/NOD_mouse'
+url2 = 'http://en.wikipedia.org/wiki/Academic_studies_about_Wikipedia'
+d1, d2 = document(url1), document(url2)
+
+# Process both of our documents.
+[d1,d2].apply(:chunk, :segment, :tokenize)
+
+# Answer our problem to create a training set.
+d1.sentences[0..17].each { |s| s.set :is_junk, 0 }
+d1.sentences[17..-1].each { |s| s.set :is_junk, 1 }
+d_set = d1.export(pb)
+
+# Define our gold standard results for evaluation.
+d2.sentences[0..81].each { |s| s.set :is_true_junk, 0 }
+d2.sentences[81..-1].each { |s| s.set :is_true_junk, 1 }
+
+tp, fp, tn, fn = 0.0, 0.0, 0.0, 0.0
+
+d2.sentences.map do |s| 
+  pred = s.classify(:id3, training: d_set)
+  if pred == 1
+    tp += 1 if s.is_true_junk == 1
+    fp += 1 if s.is_true_junk == 0
+  else
+    tn += 1 if s.is_true_junk == 0
+    fn += 1 if s.is_true_junk == 1
+  end
+end
+
+puts "Precision: #{tp/(tp + fp)}"
+puts "Recall: #{tp/(tp + fn)}"
+=end
+=begin
+d = document 'http://louismullie.com/susan-text-scan1.jpg'
+d.apply :chunk, :segment, :tokenize
+d.print_tree
+=end
+=begin 
+# Syntax example
+phra = phrase 'Obama', 'Sarkozy', 'Meeting'
+
+para  = paragraph 'Obama and Sarkozy met on January 1st to'
+'investigate the possibility of a new rescue plan. Nicolas ' +
+'Sarkozy is to meet Merkel next Tuesday in Berlin.'
+
+sect = section title(phra), para
+=end
+=begin
+puts "beer".plural.inspect
+=end
+p = paragraph

data/spec/treat.rb

@@ -2,44 +2,36 @@ require_relative 'helper'
 
 describe Treat do
 
-  describe "Syntactic sugar:"
-
-  describe "#sweeten!, #unsweeten!" do
-
-    it "respectively turn on and off syntactic sugar and " +
-    "define/undefine entity builders as uppercase methods " +
-    "in the global namespace" do
-
-      Treat.core.entities.list.each do |type|
-
-        next if type == :symbol
-
-        Treat::Config.sweeten!
-
-        Treat.core.syntax.sweetened.should eql true
-
-        Object.method_defined?(
-        :"#{type.to_s.capitalize}").
-        should eql true
-
-        Treat::Config.unsweeten!
-        Treat.core.syntax.sweetened.should eql false
-
-        Object.method_defined?(
-        type.to_s.capitalize.intern).should eql false
-
-        Object.method_defined?(
-        :"#{type.to_s.capitalize}").
-        should eql false
+  describe "Syntactic sugar:" do
+    
+    describe "#sweeten!, #unsweeten!" do
+      it "respectively turn on and off syntactic sugar and " +
+      "define/undefine entity builders as uppercase methods " +
+      "in the global namespace" do
+        Treat.core.entities.list.each do |type|
+          next if type == :symbol
+
+          Treat::Config.sweeten!
+          Treat.core.syntax.sweetened.should eql true
+          Object.method_defined?(
+          :"#{type.to_s.capitalize}").
+          should eql true
         
+          Treat::Config.unsweeten!
+          Treat.core.syntax.sweetened.should eql false
+          Object.method_defined?(
+          type.to_s.capitalize.intern).should eql false
+          Object.method_defined?(
+          :"#{type.to_s.capitalize}").
+          should eql false        
+        end
       end
-
     end
-
+    
   end
-
+  
   describe "Paths:" do
-
+    
     paths = Treat.core.paths.description
     # Check IO for bin, files, tmp, models. Fix.
     paths.each_pair do |path, files|
@@ -49,7 +41,7 @@ describe Treat do
         end
       end
     end
-
+    
   end
 
 end

data/spec/workers/agnostic.rb

@@ -0,0 +1,137 @@
+class Treat::Specs::Workers::Agnostic < Treat::Specs::Workers::Language
+
+  # TODO: :tf_idf, :keywords, :classifiers
+  # :read,. :unserialize
+
+  Scenarios = {
+
+    # Also tests unserialize.
+    serialize: {
+      entity: {
+        examples: [
+          ["A test entity.", "A test entity."]
+        ],
+        generator: lambda { |selector| Treat::Entities::Entity.build(selector).to_s }
+      }
+    },
+    classify: {
+      entity: {
+        examples: [
+          ["Homer", 1, lambda { {training: Treat::Learning::DataSet.build('test.marshal')} }]
+        ],
+        preprocessor: lambda do |entity|
+          ds = Treat::Learning::DataSet.new(
+          Treat::Learning::Problem.new(
+            Treat::Learning::Question.new(:is_person, :word, :false, :discrete), 
+            Treat::Learning::Feature.new(:first_capital, 0, "->(e) {  (e.to_s[0] =~ /^[A-Z]$/) ? 1 : 0 }"), 
+            Treat::Learning::Tag.new(:value, 0)
+          ))
+          w1, w2, w3, w4, w5 = 
+          ["Alfred", "lucky", "Hobbit", "hello", "Alice"].
+          map { |w| Treat::Entities::Word.new(w) }
+          w1.set :is_person, 1
+          w2.set :is_person, 0
+          w3.set :is_person, 1
+          w4.set :is_person, 0
+          w5.set :is_person, 1
+          ds << w1; ds << w2; ds << w3
+          ds.serialize :marshal, file: 'test.marshal'
+        end
+      }
+    },
+    visualize: {
+      entity: {
+        examples: {
+          standoff: [
+            ["I walked to the store.", "(S\n   (PRP I)   (VBD walked)   (TO to)   (DT the)   (NN store)   (. .))\n"]
+          ],
+          tree: [
+            ["I walked to the store.", "+ Sentence (*)  --- \"I walked to the store.\"  ---  {}   --- [] \n|\n+--> Word (*)  --- \"I\"  ---  {}   --- [] \n+--> Word (*)  --- \"walked\"  ---  {}   --- [] \n+--> Word (*)  --- \"to\"  ---  {}   --- [] \n+--> Word (*)  --- \"the\"  ---  {}   --- [] \n+--> Word (*)  --- \"store\"  ---  {}   --- [] \n+--> Punctuation (*)  --- \".\"  ---  {}   --- [] "]
+          ],
+          dot: [
+            ["I walked to the store.", "graph {\n* [label=\"Sentence\\n\\\"I walked to the store.\\\"\",color=\"\"]\n* [label=\"Word\\n\\\"I\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"walked\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"to\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"the\\\"\",color=\"\"]\n* -- *;\n* [label=\"Word\\n\\\"store\\\"\",color=\"\"]\n* -- *;\n* [label=\"Punctuation\\n\\\".\\\"\",color=\"\"]\n* -- *;\n}"]
+          ]
+        },
+        preprocessor: lambda  { |entity| entity.tokenize },
+        generator: lambda  { |result| result.gsub(/[0-9]+/, '*') }
+      }
+    },
+
+=begin
+    keywords: {
+      document: {
+        examples: [
+          ["./spec/workers/examples/english/economist/saving_the_euro.odt",
+            ["crisis", "government", "called", "financial", "funds", "treaty"]]
+          ],
+          preprocessor: lambda do |document|
+            coll = Treat::Entities::Collection.build('./spec/workers/examples/english/economist/')
+            coll << document
+            coll.apply(:chunk, :segment, :tokenize, :keywords)
+            document
+          end
+        },
+        section: {
+          examples: [
+            ["A test phrase", ["A", "test", "phrase"]]
+          ]
+        },
+        zone: {
+          examples: [
+            ["A test phrase", ["A", "test", "phrase"]]
+          ]
+        }
+      },
+=end
+=begin  
+    unserialize: {
+      examples: [
+        ["A test entity.", "A test entity."]
+      ],
+      generator: lambda { |selector| Treat::Entities::Entity.build(selector).to_s }
+    },
+=end
+=begin
+      # Index
+      search: {
+        collection: {
+          examples: [
+            ["./spec/workers/examples/english/economist/", 
+              "Hungary's troubles", {query: 'Hungary'}]
+          ],
+          generator: lambda { |docs| docs[0].titles[0] },
+          preprocessor: lambda { |coll| coll.apply(:index) }
+      },
+    },
+=end
+=begin
+    keywords: {
+      document: {
+        examples: [
+          ["./spec/languages/english/economist/saving_the_euro.odt", 
+            ["A", "test", "phrase"]]
+        ],
+        preprocessor: lambda { |doc| doc.parent = Collection('./spec/languages/english/economist/') }
+      },
+      section: {
+        examples: [
+          ["A test phrase", ["A", "test", "phrase"]]
+        ]
+      },
+      zone: {
+        examples: [
+          ["A test phrase", ["A", "test", "phrase"]]
+        ]
+      }
+    },
+=end
+    topic_words: {
+      collection: {
+        examples: [
+          ["./spec/workers/examples/english/economist", [["orban", "minister", "bajnai", "mr", "government", "president", "law", "brussels", "commission", "hu"], ["government", "minister", "fidesz", "mr", "hvg", "today", "hungarian", "bajnai", "national", "office"], ["mr", "today", "central", "minister", "crisis", "prime", "president", "bank", "european", "government"], ["sarkozy", "mr", "greece", "german", "summit", "france", "merkel", "opera", "growth", "euro"], ["central", "hand", "minister", "week", "bank", "forced", "hungarian", "parliament", "political", "hvg"], ["minister", "crisis", "central", "bank", "hand", "law", "forced", "bajnai", "parliament", "president"], ["mr", "bank", "european", "central", "government", "called", "today", "financial", "policies", "press"], ["mr", "crisis", "government", "central", "today", "funds", "president", "issues", "bank", "called"], ["mr", "crisis", "minister", "today", "european", "prime", "financial", "president", "issues", "treaty"], ["central", "minister", "mr", "bajnai", "orban", "bank", "parliament", "week", "fidesz", "washington"], ["mr", "central", "government", "crisis", "minister", "orban", "hand", "fidesz", "bajnai", "judicial"], ["mr", "sarkozy", "chancellor", "government", "european", "merkozy", "role", "mrs", "interest", "quickly"], ["mr", "orban", "government", "crisis", "hungarian", "independence", "prime", "today", "hand", "bajnai"], ["euro", "fiscal", "merkel", "mrs", "sarkozy", "mr", "european", "zone", "leaders", "chancellor"], ["mr", "bank", "crisis", "financial", "president", "funds", "government", "treaty", "central", "part"], ["mr", "central", "minister", "crisis", "prime", "european", "government", "bank", "treaty", "issues"], ["sarkozy", "fiscal", "merkel", "mrs", "growth", "zone", "german", "role", "paper", "quickly"], ["mr", "government", "orban", "bank", "bajnai", "hungarian", "prime", "-", "hu", "commission"], ["mr", "orban", "today", "bank", "minister", "national", "government", "-", "crisis", "forced"], ["role", "summit", "merkel", "euro", "zone", "german", "mr", "greece", "sarkozy", "step"]]]
+        ]
+      }
+    }
+  }
+
+end