treat 1.1.2 → 1.2.0

data/LICENSE

@@ -1,4 +1,4 @@
-Treat - Text Retrieval, Extraction and Annotation Toolkit, v. 1.1.1
+Treat - Text Retrieval, Extraction and Annotation Toolkit, v. 1.1.2
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

data/lib/treat/config/core/acronyms.rb

@@ -1,4 +1,5 @@
 ['xml', 'html', 'txt', 'odt', 
  'abw', 'doc', 'yaml', 'uea', 
  'lda', 'pdf', 'ptb', 'dot', 
- 'ai', 'id3', 'svo', 'mlp' ]
+ 'ai', 'id3', 'svo', 'mlp', 
+ 'svm']

data/lib/treat/config/libraries/punkt.rb

@@ -0,0 +1 @@
+{model_path: nil}

data/lib/treat/config/libraries/reuters.rb

@@ -0,0 +1 @@
+{model_path: nil}

data/lib/treat/core/data_set.rb

@@ -3,11 +3,6 @@
 # have already been classified, complete with
 # references to these entities.
 class Treat::Core::DataSet
-
-  # Used to serialize Procs.
-  silence_warnings do
-    require 'sourcify'
-  end
   
   # The classification problem this
   # data set holds data for.
@@ -15,68 +10,143 @@ class Treat::Core::DataSet
   # Items that have been already 
   # classified (training data).
   attr_accessor :items
-  # References to the IDs of the
-  # original entities contained
-  # in the data set.
-  attr_accessor :entities
   
-  # Initialize the DataSet. Can be 
-  # done with a Problem entity
-  # (thereby creating an empty set)
-  # or with a filename (representing
-  # a serialized data set which will
-  # then be deserialized and loaded).
-  def initialize(prob_or_file)
-    if prob_or_file.is_a?(String)
-      ds = self.class.
-      unserialize(prob_or_file)
-      @problem = ds.problem
-      @items = ds.items
-      @entities = ds.entities
-    else
-      @problem = prob_or_file
-      @items, @entities = [], []
+  # Initialize the DataSet.
+  def initialize(problem)
+    unless problem.is_a?(Treat::Core::Problem)
+      raise Treat::Exception, "The first argument " +
+      "to initialize should be an instance of " +
+      "Treat::Core::Problem."
+    end
+    @problem, @items = problem, []
+  end
+  
+  def self.build(from)
+    if from.is_a?(Hash)
+      Treat::Core::DataSet.unserialize(
+      Treat.databases.default.adapter, from)
+    elsif from.is_a?(String)
+      unless File.readable?(from)
+        raise Treat::Exception,
+        "Attempting to initialize data set from "
+        "file #{from}, but it is not readable."
+      end
+      Treat::Core::DataSet.unserialize(
+      extension, file: from)
     end
   end
   
-  # Add an entity to the data set.
-  # The entity's relevant features
-  # are calculated based on the 
-  # classification problem, and a
-  # line with the results of the  
-  # calculation is added to the 
-  # data set, along with the ID
-  # of the entity.
+  # Add an entity to the data set. The 
+  # entity's relevant features are 
+  # calculated based on the classification 
+  # problem, and a line with the results 
+  # of the calculation is added to the 
+  # data set, along with the ID of the entity.
   def <<(entity)
-    @items << @problem.
-    export_item(entity)
-    @entities << entity.id
+    @items << { 
+    tags: (!@problem.tags.empty? ? 
+    @problem.export_tags(entity) : []),
+    features: @problem.
+    export_features(entity),
+    id: entity.id }
+  end
+  
+  # Serialize the data set to a file,
+  # or store it inside the database.
+  def serialize(handler, options = {})
+    send("to_#{handler}", options)
   end
   
-  # Marshal the data set to the supplied
-  # file name. Marshal is used for speed;
-  # other serialization options may be
-  # provided in later versions. This 
-  # method relies on the sourcify gem
-  # to transform Feature procs to strings,
-  # since procs/lambdas can't be serialized.
-  def serialize(file)
+  # Unserialize a data set file created 
+  # by using the #serialize method.
+  def self.unserialize(handler, options)
+    self.send("from_#{handler}", options)
+  end
+  
+  # Serialize the data set using Marshal.
+  def to_marshal(options)
+    file = options[:file]
     problem = @problem.dup
     problem.features.each do |feature|
-      next unless feature.proc
-      feature.proc = feature.proc.to_source
+      feature.proc = nil
+    end
+    problem.tags.each do |tag|
+      tag.proc = nil
     end
-    data = [problem, @items, @entities]
+    data = [problem, @items]
     File.open(file, 'w') do |f| 
       f.write(Marshal.dump(data))
     end
+  end
+  
+  # Unserialize the data using Marshal.
+  def self.from_marshal(options)
+    file = options[:file]
+    data = Marshal.load(File.binread(file))
+    problem, items = *data
     problem.features.each do |feature|
-      next unless feature.proc
-      source = feature.proc[5..-1]
-      feature.proc = eval("Proc.new #{source}")
+      next unless feature.proc_string
+      feature.proc = eval(feature.proc_string)
+    end
+    problem.tags.each do |tag|
+      next unless tag.proc_string
+      tag.proc = eval(tag.proc_string)
+    end
+    data_set = Treat::Core::DataSet.new(problem)
+    data_set.items = items
+    data_set
+  end
+  
+  # Serialize the data set to a MongoDB record.
+  def to_mongo(options)
+    require 'mongo'
+    host = options[:host] || Treat.databases.mongo.host
+    db = options[:db] || Treat.databases.mongo.db
+    # UNLESS HOST, UNLESS DB
+    database = Mongo::Connection.new(host).db(db)
+    database.collection('problems').update(
+    {id: @problem.id}, @problem.to_hash, {upsert: true})
+    feature_labels = @problem.feature_labels
+    feature_labels << @problem.question.name
+    tag_labels = @problem.tag_labels
+    tags = @problem.tags.map  { |t| t.name }
+    data = database.collection('data')
+    pid = @problem.id
+    @items.each do |item|
+      item[:features] = Hash[feature_labels.zip(item[:features])]
+      item[:tags] = Hash[tag_labels.zip(item[:tags])]
+      item[:problem] = pid
+      data.insert(item)
     end
   end
   
+  def self.from_mongo(options)
+    require 'mongo'
+    host = options.delete(:host) || Treat.databases.mongo.host
+    db = options.delete(:db) || Treat.databases.mongo.db
+    database = Mongo::Connection.new(host).db(db)
+    p_record = database.collection('problems').
+    find_one(id: options[:problem])
+    unless p_record
+      raise Treat::Exception, 
+      "Couldn't retrieve problem ID #{options[:problem]}."
+    end
+    problem = Treat::Core::Problem.from_hash(p_record)
+    data = database.collection('data').find(options).to_a
+    items = []
+    data.each do |datum|
+      datum.delete("_id"); datum.delete('problem')
+      item = {}
+      item[:features] = datum['features'].values
+      item[:tags] = datum['tags'].values
+      item[:id] = datum['id']
+      items << item
+    end
+    data_set = Treat::Core::DataSet.new(problem)
+    data_set.items = items
+    data_set
+  end
+
   # Merge another data set into this one.
   def merge(data_set)
     if data_set.problem != @problem
@@ -84,25 +154,14 @@ class Treat::Core::DataSet
       "Cannot merge two data sets that " +
       "don't reference the same problem." 
     else
-      @items << data_set.items
-      @entities << data_set.entities
+      @items += data_set.items
     end
   end
   
-  # Unserialize a data set file created 
-  # by using the #serialize method.
-  def self.unserialize(file)
-    data = Marshal.load(File.binread(file))
-    problem, items, entities = *data
-    problem.features.each do |feature|
-      next unless feature.proc
-      source = feature.proc[5..-1]
-      feature.proc = eval("Proc.new #{source}")
-    end
-    data_set = Treat::Core::DataSet.new(problem)
-    data_set.items = items
-    data_set.entities = entities
-    data_set
+  # Compare with other data set.
+  def ==(data_set)
+    @problem == data_set.problem &&
+    @items == data_set.items
   end
   
 end

data/lib/treat/core/export.rb

@@ -0,0 +1,59 @@
+# Represents a feature to be used
+# in a classification task.
+class Treat::Core::Export
+  
+  # The name of the feature. If no 
+  # proc is supplied, this assumes
+  # that the target of your classification
+  # problem responds to the method
+  # corresponding to this name.
+  attr_reader :name
+  # The feature's default value, if nil.
+  attr_reader :default
+  # A proc that can be used to perform
+  # calculations before storing a feature.
+  attr_accessor :proc
+  # The proc as a string value.
+  attr_accessor :proc_string
+  
+  # Initialize a feature for a classification problem.
+  def initialize(name, default = nil, proc_string = nil)
+    unless name.is_a?(Symbol)
+      raise Treat::Exception,
+      "The first argument to initialize should "+
+      "be a symbol representing the name of the export."
+    end
+    if proc_string && !proc_string.is_a?(String)
+      raise Treat::Exception,
+      "The third argument to initialize, " +
+      "if supplied, should be a string that " +
+      "can be evaluated to yield a Proc."
+    end
+    @name, @default, @proc_string =
+    name, default, proc_string
+    begin
+      @proc = proc_string ? 
+      eval(proc_string) : nil
+    rescue Exception => e
+      raise Treat::Exception,
+      "The third argument to initialize " +
+      "did not evaluate without errors " +
+      "(#{e.message})."
+    end
+    if @proc && !@proc.is_a?(Proc)
+      raise Treat::Exception,
+      "The third argument did not evaluate to a Proc."
+    end
+  end
+  
+  # Custom comparison operator for features.
+  def ==(feature)
+    @name == feature.name &&
+    @default == feature.default &&
+    @proc_string == feature.proc_string
+  end
+  
+end
+
+class Treat::Core::Feature < Treat::Core::Export; end
+class Treat::Core::Tag < Treat::Core::Export; end

data/lib/treat/core/problem.rb

@@ -4,27 +4,56 @@
 #   to attempt to answer that question?
 class Treat::Core::Problem
 
+  # A unique identifier for the problem.
+  attr_accessor :id
   # The question we are trying to answer.
   attr_reader :question
   # An array of features that will be 
   # looked at in trying to answer the
   # problem's question.
   attr_reader :features
+  attr_reader :tags
   # Just the labels from the features.
-  attr_reader :labels
+  attr_reader :feature_labels
+  attr_reader :tag_labels
   
   # Initialize the problem with a question
-  # and an arbitrary number of features.
-  def initialize(question, *features)
-    @question = question
-    @features = features
-    @labels = @features.map { |f| f.name }
+  # and an arbitrary number of features.        # FIXME: init with id!?
+  def initialize(question, *exports)
+    unless question.is_a?(Treat::Core::Question)
+      raise Treat::Exception,
+      "The first argument to initialize " +
+      "should be an instance of " +
+      "Treat::Core::Question."
+    end
+    if exports.any? { |f| !f.is_a?(Treat::Core::Export) }
+      raise Treat::Exception,
+      "The second argument and all subsequent ones " +
+      "to initialize should be instances of subclasses " +
+      "of Treat::Core::Export."
+    end
+    @question, @id = question, object_id
+    @features = exports.select do |exp|
+      exp.is_a?(Treat::Core::Feature)
+    end
+    if @features.size == 0
+      raise Treat::Exception, 
+      "Problem should be supplied with at least "+
+      "one feature to work with."
+    end
+    @tags = exports.select do |exp|
+      exp.is_a?(Treat::Core::Tag)
+    end
+    @feature_labels = @features.map { |f| f.name }
+    @tag_labels = @tags.map { |t| t.name }
   end
   
   # Custom comparison for problems.
+  # Should we check for ID here ? FIXME
   def ==(problem)
     @question == problem.question &&
-    @features == problem.features
+    @features == problem.features &&
+    @tags == problem.tags
   end
 
   # Return an array of all the entity's
@@ -32,18 +61,72 @@ class Treat::Core::Problem
   # If include_answer is set to true, will
   # append the answer to the problem after
   # all of the features.
-  def export_item(e, include_answer = true)
-    line = []
-    @features.each do |feature|
-      r = feature.proc ? 
-      feature.proc.call(e) : 
-      e.send(feature.name)
-      line << (r || feature.default)
-    end
-    return line unless include_answer
-    line << (e.has?(@question.name) ? 
+  def export_features(e, include_answer = true)
+    features = export(e, @features)
+    return features unless include_answer
+    features << (e.has?(@question.name) ? 
     e.get(@question.name) : @question.default)
-    line
+    features
+  end
+  
+  def export_tags(entity)
+    if @tags.empty?
+      raise Treat::Exception,
+      "Cannot export the tags, because " +
+      "this problem doesn't have any."
+    end
+    export(entity, @tags)
+  end
+
+  def export(entity, exports)
+    unless @question.target == entity.type
+      raise Treat::Exception, 
+      "This classification problem targets #{@question.target}s, " +
+      "but a(n) #{entity.type} was passed to export instead."
+    end
+    ret = []
+    exports.each do |export|
+      r = export.proc ? 
+      export.proc.call(entity) : 
+      entity.send(export.name)
+      ret << (r || export.default)
+    end
+    ret
+  end
+  
+  def to_hash
+    {'question' => object_to_hash(@question),
+    'features' => @features.map { |f| 
+    object_to_hash(f.tap { |f| f.proc = nil }) },
+    'tags' => @tags.map { |t| 
+    object_to_hash(t.tap { |t| t.proc = nil }) },
+    'id' => @id }
+  end
+  
+  def self.from_hash(hash)
+    question = Treat::Core::Question.new(
+      hash['question']['name'], 
+      hash['question']['target'],
+      hash['question']['type'],
+      hash['question']['default'],
+      hash['question']['labels']
+    )
+    features = []
+    hash['features'].each do |feature|
+      features << Treat::Core::Feature.new(
+      feature['name'], feature['default'],
+      feature['proc_string'])
+    end
+    tags = []
+    hash['tags'].each do |tag|
+      tags << Treat::Core::Tag.new(
+      tag['name'], tag['default'],
+      tag['proc_string'])
+    end
+    features_and_tags = features + tags
+    p = Treat::Core::Problem.new(question, *features_and_tags)
+    p.id = hash['id']
+    p
   end
 
 end

data/lib/treat/core/question.rb

@@ -8,20 +8,35 @@ class Treat::Core::Question
   # also be used as the annotation name
   # for the answer to the question.
   attr_reader :name
-  # Can be :continuous or :discrete,
-  # depending on the features used.
-  attr_reader :type
   # Defines the target of the question
   # (e.g. :sentence, :paragraph, etc.)
   attr_reader :target
+  # Can be :continuous or :discrete,
+  # depending on the features used.
+  attr_reader :type
   # Default for the answer to the question.
   attr_reader :default
+  # A list of possible answers to the question.
+  attr_reader :labels
   
   # Initialize the question.
   def initialize(name, target, 
-    type = :continuous, default = nil)
-    @name, @target = name, target
-    @type, @default = type, default
+    type = :continuous, default = nil, labels = [])
+    unless name.is_a?(Symbol)
+      raise Treat::Exception, 
+      "Question name should be a symbol."
+    end
+    unless Treat.core.entities.list.include?(target)
+      raise Treat::Exception, "Target type should be " +
+      "a symbol and should be one of the following: " +
+      Treat.core.entities.list.inspect
+    end
+    unless [:continuous, :discrete].include?(type)
+      raise Treat::Exception, "Type should be " +
+      "continuous or discrete."
+    end
+    @name, @target, @type, @default, @labels = 
+     name,  target,  type,  default,  labels
   end
 
   # Custom comparison operator for questions.
@@ -29,7 +44,8 @@ class Treat::Core::Question
     @name == question.name &&
     @type == question.type &&
     @target == question.target &&
-    @default == question.default
+    @default == question.default &&
+    @labels = question.labels
   end
   
 end

data/lib/treat/entities/abilities/iterable.rb

@@ -94,10 +94,14 @@ module Treat::Entities::Abilities::Iterable
   end
   
   # Number of children that have a given feature.
-  def num_children_with_feature(feature)
+  # Second variable to allow for passing value to check for.
+  def num_children_with_feature(feature, value = nil, recursive = true)
     i = 0
-    each do |c| 
-      i += 1 if c.has?(feature)
+    m = method(recursive ? :each_entity : :each)
+    m.call do |c|
+      next unless c.has?(feature)
+      i += (value == nil ? 1 : 
+      (c.get(feature) == value ? 1 : 0))
     end
     i
   end

data/lib/treat/entities/abilities/stringable.rb

@@ -42,14 +42,14 @@ module Treat::Entities::Abilities::Stringable
      if caller_method(2) == :inspect
        @id.to_s
      else
-       dependencies = []
-       @dependencies.each do |dependency|
-         dependencies <<
-         "#{dependency.target}#{dependency.type}"
+       edges = []
+       @edges.each do |edge|
+         edges <<
+         "#{edge.target}#{edge.type}"
        end
        s += "  --- #{short_value.inspect}" +
        "  ---  #{@features.inspect} " +
-       "  --- #{dependencies.inspect} "
+       "  --- #{edges.inspect} "
      end
      s
    end

data/lib/treat/entities/collection.rb

@@ -6,12 +6,16 @@ module Treat::Entities
     # containing the texts of the collection.
     def initialize(folder = nil, id = nil)
       super('', id)
-      if folder && !FileTest.directory?(folder)
-        FileUtils.mkdir(folder)
+      if folder
+        if !FileTest.directory?(folder)
+          FileUtils.mkdir(folder) 
+        end
+        set :folder, folder if folder
+        i = folder + '/.index'
+        if FileTest.directory?(i)
+          set :index, i
+        end
       end
-      set :folder, folder if folder
-      i = folder + '/.index'
-      set :index, i if FileTest.directory?(i)
     end
 
     # Works like the default <<, but if the
@@ -19,7 +23,7 @@ module Treat::Entities
     # document, then copy that collection or
     # document into this collection's folder.
     def <<(entities, copy = true)
-      unless entities.is_a? Array
+      unless entities.is_a?(Array)
         entities = [entities]
       end
       entities.each do |entity|

data/lib/treat/entities/entity.rb

@@ -9,7 +9,7 @@ module Treat::Entities
 
   require 'birch'
 
-  class Entity < Treat::Core::Node
+  class Entity < ::Birch::Tree
 
     # A Symbol representing the lowercase
     # version of the class name.

data/lib/treat/helpers/objtohash.rb

@@ -0,0 +1,8 @@
+def object_to_hash(obj)
+  hash = {}
+  obj.instance_variables.each do |var|
+    val = obj.instance_variable_get(var)
+    hash[var.to_s.delete("@")] = val
+  end
+  hash
+end

data/lib/treat/loaders/stanford.rb

@@ -9,18 +9,20 @@ class Treat::Loaders::Stanford
   def self.load(language = nil)
     return if @@loaded
     language ||= Treat.core.language.default
-    jar_path = Treat.libraries.
-    stanford.jar_path || Treat.paths.bin
-    models_path = Treat.libraries.
-    stanford.model_path || Treat.paths.models
-    StanfordCoreNLP.jar_path =
-    "#{jar_path}stanford/"
-    StanfordCoreNLP.model_path =
-    "#{models_path}stanford/"
+    
+    StanfordCoreNLP.jar_path = 
+    Treat.libraries.stanford.jar_path || 
+    Treat.paths.bin + 'stanford/'
+    
+    StanfordCoreNLP.model_path = 
+    Treat.libraries.stanford.model_path || 
+    Treat.paths.models + 'stanford/'
+    
     StanfordCoreNLP.use(language)
     if Treat.core.verbosity.silence
       StanfordCoreNLP.log_file = NULL_DEVICE
     end
+    
     StanfordCoreNLP.bind
     @@loaded = true
   end

data/lib/treat/version.rb

@@ -1,3 +1,3 @@
 module Treat
-  VERSION = "1.1.2"
+  VERSION = "1.2.0"
 end

data/lib/treat/workers/formatters/serializers/mongo.rb

@@ -25,7 +25,7 @@ class Treat::Workers::Formatters::Serializers::Mongo
 
     @@database ||= Mongo::Connection.
     new(Treat.databases.mongo.host).
-    db(Treat.databases.mongo.db || options[:db])
+    db(options[:db] || Treat.databases.mongo.db)
 
     supertype =  cl(Treat::Entities.const_get(
     entity.type.to_s.capitalize.intern).superclass).downcase
@@ -35,7 +35,7 @@ class Treat::Workers::Formatters::Serializers::Mongo
     coll = @@database.collection(supertypes)
     
     if entity.type == :collection
-      docs = @@database.collection('documents')
+      docs = @@database.collection('documents')                    # Take a design decision here.
       coll.update(
         {id: entity.id}, self.do_serialize(entity, 
         options.merge({:stop_at => Treat::Entities::Document})),