treat 1.1.1 → 1.1.2

data/lib/treat/config/databases/default.rb

@@ -0,0 +1 @@
+{adapter: :mongo}

data/lib/treat/config/databases/mongo.rb

@@ -1,3 +1 @@
-{host: 'localhost',
- port: '27017',
- db: nil }
+{host: 'localhost', port: '27017', db: nil }

data/lib/treat/core/data_set.rb

@@ -77,10 +77,22 @@ class Treat::Core::DataSet
     end
   end
   
+  # Merge another data set into this one.
+  def merge(data_set)
+    if data_set.problem != @problem
+      raise Treat::Exception,
+      "Cannot merge two data sets that " +
+      "don't reference the same problem." 
+    else
+      @items << data_set.items
+      @entities << data_set.entities
+    end
+  end
+  
   # Unserialize a data set file created 
   # by using the #serialize method.
   def self.unserialize(file)
-    data = Marshal.load(File.read(file))
+    data = Marshal.load(File.binread(file))
     problem, items, entities = *data
     problem.features.each do |feature|
       next unless feature.proc

data/lib/treat/core/feature.rb

@@ -7,12 +7,12 @@ class Treat::Core::Feature
   # that the target of your classification
   # problem responds to the method
   # corresponding to this name.
-  attr_accessor :name
+  attr_reader :name
   # A proc that can be used to perform
   # calculations before storing a feature.
   attr_accessor :proc
   # The default value to be
-  attr_accessor :default
+  attr_reader :default
   
   # Initialize a feature for a classification
   # problem. If two arguments are supplied, 
@@ -32,4 +32,11 @@ class Treat::Core::Feature
     end
   end
   
+  # Custom comparison operator for features.
+  def ==(feature)
+    @name == feature.name &&
+    @proc == feature.proc &&
+    @default == feature.default
+  end
+  
 end

data/lib/treat/core/problem.rb

@@ -5,13 +5,13 @@
 class Treat::Core::Problem
 
   # The question we are trying to answer.
-  attr_accessor :question
+  attr_reader :question
   # An array of features that will be 
   # looked at in trying to answer the
   # problem's question.
-  attr_accessor :features
+  attr_reader :features
   # Just the labels from the features.
-  attr_accessor :labels
+  attr_reader :labels
   
   # Initialize the problem with a question
   # and an arbitrary number of features.
@@ -20,6 +20,12 @@ class Treat::Core::Problem
     @features = features
     @labels = @features.map { |f| f.name }
   end
+  
+  # Custom comparison for problems.
+  def ==(problem)
+    @question == problem.question &&
+    @features == problem.features
+  end
 
   # Return an array of all the entity's
   # features, as defined by the problem.

data/lib/treat/core/question.rb

@@ -7,15 +7,15 @@ class Treat::Core::Question
   # (e.g. is_key_sentence), which will
   # also be used as the annotation name
   # for the answer to the question.
-  attr_accessor :name
+  attr_reader :name
   # Can be :continuous or :discrete,
   # depending on the features used.
-  attr_accessor :type
+  attr_reader :type
   # Defines the target of the question
   # (e.g. :sentence, :paragraph, etc.)
-  attr_accessor :target
+  attr_reader :target
   # Default for the answer to the question.
-  attr_accessor :default
+  attr_reader :default
   
   # Initialize the question.
   def initialize(name, target, 
@@ -24,4 +24,12 @@ class Treat::Core::Question
     @type, @default = type, default
   end
 
+  # Custom comparison operator for questions.
+  def ==(question)
+    @name == question.name &&
+    @type == question.type &&
+    @target == question.target &&
+    @default == question.default
+  end
+  
 end

data/lib/treat/entities/abilities/buildable.rb

@@ -4,11 +4,11 @@
 # is pretty much self-explanatory.
 # FIXME how can we make this language independent?
 module Treat::Entities::Abilities::Buildable
-  
+
   require 'schiphol'
   require 'fileutils'
   require 'uri'
-  
+
   # Simple regexps to match common entities.
   WordRegexp = /^[[:alpha:]\-']+$/
   NumberRegexp = /^#?([0-9]+)(\.[0-9]+)?$/
@@ -16,7 +16,7 @@ module Treat::Entities::Abilities::Buildable
   UriRegexp = /^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(([0-9]{1,5})?\/.*)?$/ix
   EmailRegexp = /.+\@.+\..+/
   Enclitics = %w['ll 'm 're 's 't 've]
-  
+
   # Reserved folder names
   Reserved = ['.index']
 
@@ -26,8 +26,11 @@ module Treat::Entities::Abilities::Buildable
   def build(file_or_value, options = {})
 
     fv = file_or_value.to_s
-    if self == Treat::Entities::Document || 
-      (fv.index('yml') || fv.index('yaml') || 
+
+    if file_or_value.is_a?(Hash)
+      from_db(file_or_value)
+    elsif self == Treat::Entities::Document ||
+      (fv.index('yml') || fv.index('yaml') ||
       fv.index('xml') || fv.index('mongo'))
       if fv =~ UriRegexp
         from_url(fv, options)
@@ -52,7 +55,7 @@ module Treat::Entities::Abilities::Buildable
         "filename, string or number."
       end
     end
-    
+
   end
 
   # Build an entity from a string. Type is
@@ -76,7 +79,7 @@ module Treat::Entities::Abilities::Buildable
     end
 
     e
-    
+
   end
 
   # Build a document from an URL.
@@ -87,14 +90,14 @@ module Treat::Entities::Abilities::Buildable
       'Cannot create something ' +
       'else than a document from a url.'
     end
-    
+
     f = Schiphol.download(url,
-      :download_folder => Treat.paths.files,
-      :show_progress => Treat.core.verbosity.silence,
-      :rectify_extensions => true,
-      :max_tries => 3
+    :download_folder => Treat.paths.files,
+    :show_progress => Treat.core.verbosity.silence,
+    :rectify_extensions => true,
+    :max_tries => 3
     )
-    
+
     options[:default_to] ||= 'html'
 
     e = from_file(f, options)
@@ -123,7 +126,7 @@ module Treat::Entities::Abilities::Buildable
   def from_folder(folder, options)
 
     return if Reserved.include?(folder)
-    
+
     unless FileTest.directory?(folder)
       raise Treat::Exception,
       "Path '#{folder}' does " +
@@ -163,9 +166,9 @@ module Treat::Entities::Abilities::Buildable
   # Build a document from a raw or serialized file.
   def from_file(file, options)
 
-    if file.index('yml') || 
-      file.index('yaml') || 
-      file.index('xml') || 
+    if file.index('yml') ||
+      file.index('yaml') ||
+      file.index('xml') ||
       file.index('mongo')
       from_serialized_file(file, options)
     else
@@ -174,7 +177,7 @@ module Treat::Entities::Abilities::Buildable
       options[:_format] = fmt
       from_raw_file(file, options)
     end
-    
+
   end
 
   # Build a document from a raw file.
@@ -186,13 +189,13 @@ module Treat::Entities::Abilities::Buildable
       "Cannot create something else than a " +
       "document from raw file '#{file}'."
     end
-    
+
     unless File.readable?(file)
       raise Treat::Exception,
       "Path '#{file}' does not "+
       "point to a readable file."
     end
-    
+
     d = Treat::Entities::Document.new(file)
 
     d.read(:autoselect, options)
@@ -217,13 +220,20 @@ module Treat::Entities::Abilities::Buildable
       d.children[0].set_as_root!              # Fix this
       d.children[0]
     end
-    
+
   end
-  
-  def from_db(adapter, options)
-    id = options[:id]
-    e = self.new(nil, id)
-    e.unserialize(adapter, options)
+
+  def from_db(hash)
+    adapter = (hash.delete(:adapter) ||
+    Treat.databases.default.adapter)
+    unless adapter
+      raise Treat::Exception,
+      "You must supply which database " +
+      "adapter to use by passing the :adapter " +
+      "option or setting configuration option" +
+      "Treat.databases.default.adapter"
+    end
+    self.new.unserialize(adapter, hash)
   end
 
   # Build any kind of entity from a string.
@@ -321,10 +331,11 @@ module Treat::Entities::Abilities::Buildable
     end
 
   end
-
+  
   def create_collection(fv)
     FileUtils.mkdir(fv)
     Treat::Entities::Collection.new(fv)
   end
 
+
 end

data/lib/treat/entities/abilities/debuggable.rb

@@ -58,7 +58,7 @@ module Treat::Entities::Abilities::Debuggable
       worker.to_s.gsub('_', ' ')
     end
     
-    curr.gsub!('ss', 's')
+    curr.gsub!('ss', 's') unless curr.index('class')
     curr += '.'
     
     if curr == @@prev

data/lib/treat/entities/collection.rb

@@ -6,7 +6,10 @@ module Treat::Entities
     # containing the texts of the collection.
     def initialize(folder = nil, id = nil)
       super('', id)
-      set :folder, folder
+      if folder && !FileTest.directory?(folder)
+        FileUtils.mkdir(folder)
+      end
+      set :folder, folder if folder
       i = folder + '/.index'
       set :index, i if FileTest.directory?(i)
     end
@@ -21,11 +24,13 @@ module Treat::Entities
       end
       entities.each do |entity|
         if [:document, :collection].
-          include?(entity.type) && copy
+          include?(entity.type) && copy &&
+          @features[:folder] != nil
           entity = entity.copy_into(self)
         end
       end
       super(entities)
     end
+    
   end
 end

data/lib/treat/version.rb

@@ -1,3 +1,3 @@
 module Treat
-  VERSION = "1.1.1"
+  VERSION = "1.1.2"
 end

data/lib/treat/workers/formatters/serializers/mongo.rb

@@ -31,10 +31,25 @@ class Treat::Workers::Formatters::Serializers::Mongo
     entity.type.to_s.capitalize.intern).superclass).downcase
     supertype = entity.type.to_s if supertype == 'entity'
     supertypes = supertype + 's'
-      
+    
     coll = @@database.collection(supertypes)
-    entity_token = self.do_serialize(entity, options)
-    coll.update({id: entity.id}, entity_token, {upsert: true})
+    
+    if entity.type == :collection
+      docs = @@database.collection('documents')
+      coll.update(
+        {id: entity.id}, self.do_serialize(entity, 
+        options.merge({:stop_at => Treat::Entities::Document})), 
+        {upsert: true})
+      entity.each_document do |doc|
+        docs.update(
+        {id: doc.id}, self.do_serialize(doc, options), 
+        {upsert: true})
+      end
+    else
+      entity_token = self.do_serialize(entity, options)
+      coll.update({id: entity.id}, entity_token, {upsert: true})
+    end
+    
   end
 
   def self.do_serialize(entity, options)
@@ -52,7 +67,6 @@ class Treat::Workers::Formatters::Serializers::Mongo
     entity_token = {
       :id => entity.id,
       :value => entity.value,
-      :string => entity.to_s,
       :type => entity.type.to_s,
       :children => children,
       :parent => (entity.has_parent? ? entity.parent.id : nil),

data/lib/treat/workers/formatters/unserializers/mongo.rb

@@ -3,8 +3,11 @@ module Treat::Workers::Formatters::Unserializers::Mongo
   require 'mongo'
   
   def self.unserialize(entity, options={})
-
-    if !Treat.databases.mongo.db && !options[:db]
+    
+    db = options.delete(:db)
+    selector = options
+    
+    if !Treat.databases.mongo.db && !db
       raise Treat::Exception,
       'Must supply the database name in config. ' +
       '(Treat.databases.mongo.db = ...) or pass ' +
@@ -13,7 +16,7 @@ module Treat::Workers::Formatters::Unserializers::Mongo
     
     @@database ||= Mongo::Connection.
     new(Treat.databases.mongo.host).
-    db(Treat.databases.mongo.db || options[:db])
+    db(Treat.databases.mongo.db || db)
     
     supertype =  cl(Treat::Entities.const_get(
     entity.type.to_s.capitalize.intern).superclass).downcase
@@ -21,14 +24,23 @@ module Treat::Workers::Formatters::Unserializers::Mongo
     supertypes = supertype + 's'
     
     coll = @@database.collection(supertypes)
-    record = coll.find_one(:id => entity.id)
+    records = coll.find(selector).to_a
     
-    unless record
+    if records.size == 0
       raise Treat::Exception,
-      "Couldn't find record ID #{entity.id}."
+      "Couldn't find any records using " +
+      "selector #{selector.inspect}."
+    elsif records.size == 1
+      self.do_unserialize(
+      records.first, options)
+    else
+      matches = []
+      records.each do |record|
+        matches << self.
+        do_unserialize(record, options)
+      end
+      matches
     end
-
-    self.do_unserialize(record, options)
     
   end
 

data/lib/treat/workers/processors/parsers/stanford.rb

@@ -29,7 +29,7 @@ class Treat::Workers::Processors::Parsers::Stanford
     lang = entity.language
     init(lang, options)
     
-    tag_set = StanfordCoreNLP::Config::TagSets[language]
+    tag_set = StanfordCoreNLP::Config::TagSets[lang]
     
     text = ::StanfordCoreNLP::Text.new(val)
     @@parsers[lang].annotate(text)

data/lib/treat/workers/retrievers/indexers/ferret.rb

@@ -18,6 +18,8 @@ class Treat::Workers::Retrievers::Indexers::Ferret
   # index for future use (e.g. in searching).
   def self.index(collection, options = {})
     
+    # FIXME - what if the collection is stored
+    # in a database?
     path = "#{collection.folder}/.index"
     return path if FileTest.directory?(path)
     

data/spec/core.rb

@@ -0,0 +1,32 @@
+require_relative '../lib/treat'
+
+describe Treat::Core::DataSet do
+=begin
+
+
+p = Problem(
+  Question(:is_key_sentence, :sentence, false),
+  Feature(:word_count, 0)
+)
+
+p2 = Problem(
+  Question(:is_key_sentence, :sentence, false),
+  Feature(:word_count, 0)
+)
+
+ds = DataSet(p)
+
+text = Paragraph("Welcome to the zoo! This is a text.")
+text2 = Paragraph("Welcome here my friend. This is well, a text.")
+
+text.do :segment, :tokenize
+text2.do :segment, :tokenize
+
+ds1 = text.export(p)
+ds2 = text2.export(p2)
+
+ds1.merge(ds2)
+
+puts ds1.inspect
+=end
+end

data/spec/sandbox.rb

@@ -0,0 +1,13 @@
+require_relative '../lib/treat'
+
+Treat.databases.mongo.db = 'testingyetagain'
+
+d = Document('http://www.cbc.ca/news/world/story/2012/07/21/syria-aleppo-clashes.html')
+
+d2 = Document('http://www.cbc.ca/news/canada/british-columbia/story/2012/07/19/bc-monty-robinson-sentencing.html')
+
+c = Collection('testing')
+c << d
+c << d2
+
+c.serialize :mongo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: treat
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-19 00:00:00.000000000 Z
+date: 2012-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: schiphol
@@ -106,6 +106,7 @@ files:
 - lib/treat/config/core/paths.rb
 - lib/treat/config/core/syntax.rb
 - lib/treat/config/core/verbosity.rb
+- lib/treat/config/databases/default.rb
 - lib/treat/config/databases/mongo.rb
 - lib/treat/config/languages/agnostic.rb
 - lib/treat/config/languages/arabic.rb
@@ -243,6 +244,7 @@ files:
 - lib/treat/workers.rb
 - lib/treat.rb
 - spec/collection.rb
+- spec/core.rb
 - spec/document.rb
 - spec/entity.rb
 - spec/node.rb
@@ -252,12 +254,16 @@ files:
 - spec/samples/mathematicians/gauss.pdf
 - spec/samples/mathematicians/leibniz.txt
 - spec/samples/mathematicians/newton.doc
+- spec/sandbox.rb
 - spec/token.rb
 - spec/treat.rb
 - spec/word.rb
 - spec/zone.rb
 - tmp/MANIFEST
+- files/3_2_release_notes.html
+- files/bc-monty-robinson-sentencing.html
 - files/MANIFEST
+- files/syria-aleppo-clashes.html
 - README.md
 - LICENSE
 homepage: https://github.com/louismullie/treat