feature_set 0.0.2 → 0.0.4

data/README.markdown

@@ -1,8 +1,36 @@
-## FeatureSet
+# This library is alpha and is not yet finished.
 
+# FeatureSet
 
-Helpful tasks:
+A Ruby library for building machine learning datasets.
 
-- rake build
-- rake install
-- rake release
+In machine learning, feature selection is often more difficult than algorithm selection.  For many classes of problems, any reasonably modern algorithm can be used (i.e., a SVM, decision tree, etc.).  However, all of these algorithms require information-rich features to learn from, and finding and constructing those features can is often its own engineering challenge.  FeatureSet is a library that makes it easy to construct features from your data as a pre-processing step before applying a modern machine learning library such as Weka or libsvm.
+
+FeatureSet takes a dataset consisting of hashes, with any any object as the value of each key, and builds features from these values as appropriate.  For example, a string value could be expanded into a number of new features- a count of cuss words in the string, a count of slang, a sentiment score, and/or a complete word vector with TF-IDF values.
+
+FeatureSet is extensible, so anyone can write new FeatureBuilders that know to which datatypes they can be applied.  The set of included feature builders expands as the community submits new ones.
+
+## FeatureBuilders
+
+## Example Code
+
+    data_set = FeatureSet::DataSet.new
+    data_set.add_feature_builder FeatureSet::FeatureBuilders::WordVector.new(:word_limit => 2000, :idf_cutoff => 8.0)
+    data_set.add_feature_builder FeatureSet::FeatureBuilders::Cuss.new
+    data_set.add_data :status => "This is a spam email", :class => :spam
+    data_set.add_data :status => "This is a not spam", :class => :not_spam
+    data_set.build_features_from_data!(:include_original => false) #do not include :status as it's own column in the output
+
+    # The following ARFF can be imported into Weka
+    puts data_set.to_rarff.to_s
+
+    serialized_builders = data_set.dump_feature_builders
+    
+    ... later ...
+
+    data_set = FeatureSet::DataSet.new
+    data_set.load_feature_builders(serialized_builders)
+    features = data_set.build_features_for({ :status => "Is this spam?" })
+    
+
+See the specs for more usage examples.

data/feature_set.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
 
   # specify any dependencies here; for example:
   s.add_development_dependency "rspec"
-  s.add_runtime_dependency "wwood-rarff"
+  s.add_runtime_dependency "iterationlabs-rarff"
   s.add_runtime_dependency "activesupport"
   s.add_runtime_dependency "i18n"
 end

data/lib/feature_set.rb

@@ -1,4 +1,4 @@
+require "rubygems"
 require "rarff"
-
 require "feature_set/version"
-require "feature_set/builder"
+require "feature_set/data_set"

data/lib/feature_set/data_set.rb

@@ -0,0 +1,135 @@
+require 'active_support'
+require 'active_support/inflector'
+
+require "feature_set/feature_builders/word_vector"
+require "feature_set/feature_builders/cuss"
+require "feature_set/feature_builders/emoticon"
+
+require "feature_set/datum"
+
+module FeatureSet
+  class DataSet
+    BUILTIN_FEATURE_BUILDERS = %w[FeatureSet::FeatureBuilders::Cuss 
+                                  FeatureSet::FeatureBuilders::Emoticon
+                                  FeatureSet::FeatureBuilders::WordVector].map(&:constantize)
+
+    attr_accessor :options, :feature_builders, :data, :features, :name
+
+    def initialize(options = {})
+      @options = options
+      @name = options[:name]
+      @feature_builders = []
+      @features = []
+      @data = []
+    end
+    
+    def add_data(data)
+      (@data << data).flatten!
+    end
+    
+    def clear_data
+      @data = []
+    end
+    
+    def clear_features
+      @features = []
+    end
+
+    def to_rarff
+      relation = Rarff::Relation.new(name || 'Data')
+      keys = features.first.keys
+      instances = features.map do |row|
+        keys.map do |key|
+          value = row[key]
+          if value.is_a?(String)
+            value.gsub(/\\/, "\\\\\\\\").gsub(/"/, "\\\\\"").gsub(/'/, '\\\\\'')
+          elsif value.is_a?(Symbol)
+            value.to_s
+          else
+            value
+          end
+        end
+      end
+      relation.instances = instances
+      keys.each_with_index do |key, index|
+        relation.attributes[index].name = key.to_s
+      end
+      relation
+    end
+    
+    # This only knows how to output arfs with true/false classes and all numeric attributes.
+    # Additionally, every row must have the same attributes.
+    def output_numeric_arff(io)
+      keys = features.first.keys
+      io.puts "@RELATION Data"
+      keys.each do |key|
+        io.puts "@ATTRIBUTE #{key} NUMERIC" unless key == :class
+      end
+      io.puts "@ATTRIBUTE class {false,true}"
+      io.puts "@DATA"
+      features.each do |feature|
+        io.puts keys.map { |k| k == :class ? feature[k].to_s : feature[k].to_f }.join(",")
+      end
+    end
+    
+    def build_features_from_data!(opts = {})
+      wrapped_data = self.class.wrap_dataset(data)
+      feature_builders.each {|fb| fb.before_build_features(wrapped_data) }
+      @features = build_features_for(wrapped_data, opts.merge(:already_wrapped => true))
+    end
+
+    def build_features_for(data, opts = {})
+      # FYI, we explicitly do not call before_build_features because this can be used on unknown rows for classification, and
+      # we want our feature builders to keep any cached data from the previous 'build_features_from_data!' call.  This is important for
+      # Wordvector, for example, since it needs to build the idf mappings beforehand and needs to re-use them on any new data.
+      wrapped_data = opts[:already_wrapped] ? data : self.class.wrap_dataset(data)
+      wrapped_data.map.with_index do |row, index|
+        output_row = {}
+        
+        row.each do |key, datum|
+          if key == :class
+            output_row[:class] = datum
+            next
+          end
+          
+          if opts[:include_original] && (opts[:include_original].is_a?(TrueClass) || ![opts[:include_original][:except]].flatten.include?(key))
+            output_row[key] = datum.value
+          end
+
+          feature_builders.each do |builder|
+            builder.build_features(datum, key, row).each do |feature, value|
+              output_row["#{key}_#{feature}".to_sym] = value
+            end
+          end
+        end
+        
+        if index % 10 == 0
+          STDERR.print "."; STDERR.flush
+        end
+        
+        output_row
+      end
+    end
+
+    def add_feature_builders(*builders)
+      builders = BUILTIN_FEATURE_BUILDERS.map(&:new) if [:all, "all"].include?(builders.first)
+      (@feature_builders << builders).flatten!
+    end
+    alias_method :add_feature_builder, :add_feature_builders
+    
+    
+    def dump_feature_builders
+      Marshal.dump(feature_builders)
+    end
+    
+    def load_feature_builders(serialized_builders)
+      clear_features
+      self.feature_builders = Marshal.load(serialized_builders)
+    end
+    
+    def self.wrap_dataset(dataset)
+      dataset = [dataset] unless dataset.is_a?(Array)
+      dataset.map { |row| row.inject({}) { |m, (k, v)| m[k] = (k == :class ? v : Datum.new(v)) ; m } }
+    end
+  end
+end

data/lib/feature_set/feature_builders/base.rb

@@ -0,0 +1,18 @@
+module FeatureSet
+  module FeatureBuilders
+    class Base
+      attr_accessor :options
+      
+      def initialize(options = {})
+        @options = options
+      end
+      
+      def build_features(datum, key, row)
+        raise "Please implement 'build_features' in your subclass of FeatureBuilders::Base."
+      end
+      
+      def before_build_features(dataset)
+      end
+    end
+  end
+end

data/lib/feature_set/{feature_builder → feature_builders}/cuss.rb

@@ -1,11 +1,11 @@
-require "feature_set/feature_builder/base"
+require "feature_set/feature_builders/base"
 
 module FeatureSet
-  module FeatureBuilder
+  module FeatureBuilders
     class Cuss < Base
       CUSS_WORDS = File.read(File.expand_path(File.join(File.dirname(__FILE__), '..', 'data', 'cusswords.txt'))).split("\n").map {|i| i.strip.downcase }
       
-      def generate_features(datum, key, row)
+      def build_features(datum, key, row)
         return {} unless datum.value.is_a?(String)
         { :cuss_count => (datum.tokens & CUSS_WORDS).length }
       end

data/lib/feature_set/feature_builders/emoticon.rb

@@ -0,0 +1,19 @@
+require "feature_set/feature_builders/base"
+
+module FeatureSet
+  module FeatureBuilders
+    class Emoticon < Base
+      HAPPY = [">:]", ":-)", ":)", ":o)", ":]", ":3", ":c)", ":>", "=]", "8)", "=)", ":}", ":^)", ">:D", ":-D", ":D", "8-D", "8D", "x-D", "xD", "X-D", "XD", "=-D", "=D", "=-3", "=3"]
+      SAD = [":'(", ";*(", ":_(", "T.T", "T_T", "Y.Y", "Y_Y", ">:[", ":-(", ":(", ":-c", ":c", ":-<", ":<", ":-[", ":[", ":{", ">.>", "<.<", ">.<", "D:<", "D:", "D8", "D;", "D=", "DX", "v.v", "D-':"]
+      HUMOR = [">;]", ";-)", ";)", "*-)", "*)", ";-]", ";]", ";D", ">:P", ":-P", ":P", "X-P", "x-p", "xp", "XP", ":-p", ":p", "=p", ":-b", ":b"]
+      
+      def build_features(datum, key, row)
+        return {} unless datum.value.is_a?(String)
+        tokens = datum.value.split(/\s+/)
+        { :happy_emoticon_count => (tokens & HAPPY).length,
+          :sad_emoticon_count => (tokens & SAD).length,
+          :humor_emoticon_count => (tokens & HUMOR).length }
+      end
+    end
+  end
+end

data/lib/feature_set/feature_builders/word_vector.rb

@@ -0,0 +1,85 @@
+require "feature_set/feature_builders/base"
+
+module FeatureSet
+  module FeatureBuilders
+    class WordVector < Base
+      attr_accessor :idfs
+
+      # Options:
+      #   :tf_only => true|false, default is false
+      #   :idf_cutiff => <cutoff>, default is 10
+      #   :word_limit => <word limit>, default is 2000
+      def initialize(options = {})
+        super
+        @idfs = {}
+      end
+
+      def before_build_features(dataset)
+        @idfs = {}
+        dataset.each do |row|
+          row.each do |key, datum|
+            next if key == :class
+            if datum.value.is_a?(String)
+              idfs[key] ||= {}
+              datum.token_counts.keys.each do |token|
+                idfs[key][token] ||= 0
+                idfs[key][token] += 1
+              end
+            end
+          end
+        end
+        
+        num_docs = dataset.length
+        idf_cutoff = (options[:idf_cutoff] || 10).to_f
+        word_limit = options[:word_limit] || 2000
+        STDERR.puts "Done building df counts.  The dataset has #{num_docs} documents."
+
+        idfs.each do |feature, freqs|
+          pruned = 0
+          if options[:tf_only]
+            new_freqs = freqs
+          else
+            new_freqs = {}
+            freqs.each do |key, value|
+              log = Math.log(num_docs / value.to_f)
+              if log < idf_cutoff
+                new_freqs[key] = log
+              else
+                pruned += 1
+              end
+            end
+          end
+          if options[:word_limit]
+            new_freqs = if options[:tf_only]
+                          new_freqs.to_a.sort {|a, b| b.last <=> a.last }
+                        else
+                          new_freqs.to_a.sort {|a, b| a.last <=> b.last }
+                        end
+            new_freqs = new_freqs[0...word_limit].inject({}) { |m, (k, v)| m[k] = v; m }
+          end
+          idfs[feature] = new_freqs
+          STDERR.puts "Done calculating idfs for #{feature}.  Pruned #{pruned} rare values, leaving #{idfs[feature].length} values."
+        end
+      end
+      
+      def build_features(datum, key, row)
+        return {} unless datum.value.is_a?(String)
+        num_words = datum.tokens.length.to_f
+        unless idfs[key]
+          STDERR.puts "WARNING: build_features called on untrained data in WordVector.  Are you calling 'data_set.build_features_for' without calling 'data_set.build_features_from_data!' first?"
+        end
+        if options[:tf_only]
+          (idfs[key] || {}).inject({}) do |memo, (word, idf)|
+            memo["wv_#{word}"] = ((datum.token_counts[word] || 0) / num_words)
+            memo
+          end
+        else
+          (idfs[key] || {}).inject({}) do |memo, (word, idf)|
+            memo["wv_#{word}"] = ((datum.token_counts[word] || 0) / num_words) * idf
+            memo
+          end
+        end
+      end
+    end
+  end
+end

data/lib/feature_set/version.rb

@@ -1,3 +1,3 @@
 module FeatureSet
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

data/spec/feature_set/data_set_spec.rb

@@ -0,0 +1,135 @@
+require 'spec_helper'
+
+describe FeatureSet::DataSet do
+  describe "adding feature builders" do
+    it "can add all known feature builders" do
+      data_set = FeatureSet::DataSet.new
+      data_set.add_feature_builders :all
+      data_set.feature_builders.map {|i| i.class}.should include(FeatureSet::FeatureBuilders::WordVector)
+      data_set.feature_builders.length.should == Dir[File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "feature_set", "feature_builders", "*.rb"))].length - 1
+    end
+    
+    it "can add individual feature builders" do
+      data_set = FeatureSet::DataSet.new
+      data_set.add_feature_builders FeatureSet::FeatureBuilders::WordVector.new
+      data_set.feature_builders.length.should == 1
+    end
+
+    it "can add arrays of feature builders" do
+      data_set = FeatureSet::DataSet.new
+      data_set.add_feature_builders [FeatureSet::FeatureBuilders::WordVector.new, FeatureSet::FeatureBuilders::Cuss.new]
+      data_set.feature_builders.length.should == 2
+    end
+  end
+  
+  describe "adding data" do
+    it "should accept mappings between one or more strings and their classifications" do
+      data_set = FeatureSet::DataSet.new
+      data_set.add_data [ { :status => "I am happy!", :class => :happy },
+                          { :status => "I am sad." , :class => :sad } ]
+      data_set.data.should == [ { :status => "I am happy!", :class => :happy },
+                                { :status => "I am sad." , :class => :sad } ]
+      data_set.add_data :status => "Something", :another_feature => "Something else", :class => :awesome
+      data_set.data.should == [ { :status => "I am happy!", :class => :happy },
+                                { :status => "I am sad." , :class => :sad },
+                                { :status => "Something", :another_feature => "Something else", :class => :awesome } ]
+      data_set.clear_data
+      data_set.data.should == []
+      data_set.data = [ { :status => "I am happy!", :class => :happy },
+                        { :status => "I am sad." , :class => :sad } ]
+      data_set.data.should == [ { :status => "I am happy!", :class => :happy },
+                                { :status => "I am sad." , :class => :sad } ]
+    end
+  end
+  
+  describe "generating features" do
+    before do
+      @data_set = FeatureSet::DataSet.new
+      @data_set.add_feature_builders FeatureSet::FeatureBuilders::Cuss.new
+      @data_set.add_data :status => "this is some text", :class => :awesome
+      @data_set.add_data :status => "this is some shitty text", :class => :less_awesome
+    end
+    
+    it "should output a row of features for every line of data" do
+      @data_set.build_features_from_data!
+      @data_set.features[0].should == { :status_cuss_count => 0, :class => :awesome }
+      @data_set.features[1].should == { :status_cuss_count => 1, :class => :less_awesome }
+    end
+    
+    it "should make it easy to keep the original data" do
+      @data_set.build_features_from_data!(:include_original => true)
+      @data_set.features[0].should == { :status => "this is some text", :status_cuss_count => 0, :class => :awesome }
+      @data_set.features[1].should == { :status => "this is some shitty text", :status_cuss_count => 1, :class => :less_awesome }
+    end
+
+    it "should generate features for every string" do
+      @data_set.add_data :status => "text", :foo => "more shitty text", :class => :awesome
+      @data_set.build_features_from_data!
+      @data_set.features[1].should == { :status_cuss_count => 1, :class => :less_awesome }
+      @data_set.features[2].should == { :status_cuss_count => 0, :foo_cuss_count => 1, :class => :awesome }
+    end
+
+    it "should allow generation of features on new data while leaving the old data intact" do
+      @data_set.build_features_from_data!
+      num_features = @data_set.features.length
+      @data_set.build_features_for([{ :status => "is this shitty text?" }, { :status => "foo bar" }]).should == [{ :status_cuss_count => 1 }, { :status_cuss_count => 0 }]
+      @data_set.features.length.should == num_features
+    end
+  end
+  
+  describe "serialization" do
+    it "should be able to serialize, saving all trained builders, but not the dataset" do
+      data_set = FeatureSet::DataSet.new
+      data_set.add_feature_builder FeatureSet::FeatureBuilders::WordVector.new
+      data_set.add_data :status => "this is some text", :class => :awesome
+      data_set.add_data :status => "this is some shitty text", :class => :less_awesome
+      data_set.build_features_from_data!
+      trained_rows = data_set.build_features_for([{ :status => "is this shitty text?" }, { :status => "foo bar" }])
+      serialized_builders = data_set.dump_feature_builders
+
+      data_set = FeatureSet::DataSet.new
+      data_set.add_feature_builder FeatureSet::FeatureBuilders::WordVector.new
+      untrained_rows = data_set.build_features_for([{ :status => "is this shitty text?" }, { :status => "foo bar" }])
+
+      data_set2 = FeatureSet::DataSet.new
+      data_set2.load_feature_builders(serialized_builders)
+      data_set2.data.should == []
+      rows_from_dump = data_set2.build_features_for([{ :status => "is this shitty text?" }, { :status => "foo bar" }])
+      rows_from_dump.should == trained_rows
+      rows_from_dump.should_not == untrained_rows
+    end
+  end
+
+  describe "outputing an ARFF file" do
+    before do
+      @data_set = FeatureSet::DataSet.new
+      @data_set.add_feature_builders FeatureSet::FeatureBuilders::Cuss.new
+      @data_set.add_data :status => "this is some text", :foo => 2, :class => :awesome
+      @data_set.add_data :status => "this is some shitty text", :foo => 5, :class => :less_awesome
+    end
+
+    describe "as an rarff relation" do
+      it "should return a rarff relation object" do
+        @data_set.build_features_from_data!(:include_original => { :except => :status })
+        arff = @data_set.to_rarff
+        arff.should be_a(Rarff::Relation)
+        arff.attributes.map(&:name).should =~ ["status_cuss_count", "class", "foo"]
+        arff.attributes.last.name.should == "class"
+        arff.to_s.should =~ /Data/
+        arff.to_s.should =~ /status_cuss_count/
+      end
+    end
+    
+    describe "as a numeric arff" do
+      it "should output an arff to an IO object" do
+        @data_set.build_features_from_data!(:include_original => { :except => :status })
+        io = StringIO.new
+        @data_set.output_numeric_arff(io)
+        io.rewind
+        str = io.read
+        str.should =~ /@ATTRIBUTE status_cuss_count NUMERIC/
+        str.scan(/@ATTRIBUTE class /).length.should == 1
+      end
+    end
+  end
+end

data/spec/feature_set/feature_builders/cuss_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe FeatureSet::FeatureBuilders::Cuss do
+  before do
+    @builder = FeatureSet::FeatureBuilders::Cuss.new
+  end
+
+  it "should output :cuss_count as the number of distinct cuss words found" do
+    @builder.build_features(FeatureSet::Datum.new("this fucking shit"), nil, nil).should == { :cuss_count => 2 }
+    @builder.build_features(FeatureSet::Datum.new("this fucking fucking fucking shit"), nil, nil).should == { :cuss_count => 2 }
+  end
+
+  it "should ignore non-string features" do
+    @builder.build_features(FeatureSet::Datum.new(2), nil, nil).should == {}
+  end
+end

data/spec/feature_set/feature_builders/emoticon_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe FeatureSet::FeatureBuilders::Emoticon do
+  before do
+    @builder = FeatureSet::FeatureBuilders::Emoticon.new
+  end
+
+  it "should output counts of the number of distinct emoticons of each type" do
+    @builder.build_features(FeatureSet::Datum.new("blah :) XP"), nil, nil).should == { :happy_emoticon_count => 1, :humor_emoticon_count => 1, :sad_emoticon_count => 0 }
+    @builder.build_features(FeatureSet::Datum.new("blah ;) :("), nil, nil).should == { :happy_emoticon_count => 0, :humor_emoticon_count => 1, :sad_emoticon_count => 1 }
+  end
+
+  it "should ignore non-string features" do
+    @builder.build_features(FeatureSet::Datum.new(2), nil, nil).should == {}
+  end
+end

data/spec/feature_set/feature_builders/word_vector_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+describe FeatureSet::FeatureBuilders::WordVector do
+  it "should output a named feature for every word in the dataset, after performing tfidf" do
+    builder = FeatureSet::FeatureBuilders::WordVector.new
+    dataset = [
+                { :m1 => "hello world.  hello!", :m2 => "how goes?", :class => :yes }, 
+                { :m1 => "foo world", :m2 => "how?", :class => :no }
+              ]
+    wrapped_dataset = FeatureSet::DataSet.wrap_dataset(dataset)
+    builder.before_build_features(wrapped_dataset)
+
+    builder.idfs.should == {
+                             :m1 => { "hello" => Math.log(2/1.0), "world" => Math.log(2/2.0), "foo" => Math.log(2/1.0) },
+                             :m2 => { "how" => Math.log(2/2.0), "goes" => Math.log(2/1.0) }
+                           }
+
+    builder.build_features(wrapped_dataset.first[:m1], :m1, wrapped_dataset.first).should == { "wv_hello" => (2/3.0) * Math.log(2/1.0), "wv_world" => (1/3.0) * Math.log(2/2.0), "wv_foo" => 0 }
+    builder.build_features(wrapped_dataset.first[:m2], :m2, wrapped_dataset.first).should == { "wv_how" => (1/2.0) * Math.log(2/2.0), "wv_goes" => (1/2.0) * Math.log(2/1.0) }
+
+    builder.build_features(wrapped_dataset.last[:m1], :m1, wrapped_dataset.last).should == { "wv_hello" => 0, "wv_world" => (1/2.0) * Math.log(2/2.0), "wv_foo" => (1/2.0) * Math.log(2/1.0) }
+    builder.build_features(wrapped_dataset.last[:m2], :m2, wrapped_dataset.last).should == { "wv_how" => (1/1.0) * Math.log(2/2.0), "wv_goes" => 0 }
+  end
+
+  it "should ignore non-string features" do
+    builder = FeatureSet::FeatureBuilders::WordVector.new
+    builder.before_build_features([{ :something => FeatureSet::Datum.new(2), :class => false }, { :something => FeatureSet::Datum.new(1), :class => true }])
+    builder.build_features(FeatureSet::Datum.new(2), :something, { :something => FeatureSet::Datum.new(2), :class => false }).should == {}
+  end
+  
+  it "should allow specifying the idf cutoff" do
+    builder = FeatureSet::FeatureBuilders::WordVector.new(:idf_cutoff => 2.0)
+    dataset = [{ :m1 => "hello world.  hello!", :class => true }] * 10
+    dataset <<  { :m1 => "foo", :class => false }
+    wrapped_dataset = FeatureSet::DataSet.wrap_dataset(dataset)
+    builder.before_build_features(wrapped_dataset)
+    builder.idfs.should == {
+                             :m1 => { "hello" => Math.log(11/10.0), "world" => Math.log(11/10.0) }
+                           }
+  end
+  
+  it "should allow specifying a word-count threshold" do
+    builder = FeatureSet::FeatureBuilders::WordVector.new(:word_limit => 2)
+    dataset = [{ :m1 => "hello world.  hello!", :class => true }] * 10
+    dataset <<  { :m1 => "foo", :class => false }
+    dataset <<  { :m1 => "hello", :class => false }
+    dataset <<  { :m1 => "hello", :class => false }
+    wrapped_dataset = FeatureSet::DataSet.wrap_dataset(dataset)
+    builder.before_build_features(wrapped_dataset)
+    builder.idfs.should == {
+                             :m1 => { "hello" => Math.log(13/12.0), "world" => Math.log(13/10.0) }
+                           }
+
+    builder = FeatureSet::FeatureBuilders::WordVector.new(:word_limit => 1)
+    dataset = [{ :m1 => "hello world.  hello!", :class => true }] * 10
+    dataset <<  { :m1 => "foo", :class => false }
+    dataset <<  { :m1 => "world", :class => false }
+    dataset <<  { :m1 => "world", :class => false }
+    wrapped_dataset = FeatureSet::DataSet.wrap_dataset(dataset)
+    builder.before_build_features(wrapped_dataset)
+    builder.idfs.should == {
+                             :m1 => { "world" => Math.log(13/12.0) }
+                           }
+  end
+  
+  it "should allow use of TF-only without IDF" do
+    builder = FeatureSet::FeatureBuilders::WordVector.new(:tf_only => true, :word_limit => 2)
+    dataset = [
+                { :m1 => "hello world.  hello!", :m2 => "how goes?", :class => :yes }, 
+                { :m1 => "foo world", :m2 => "how?", :class => :no },
+                { :m1 => "hello world!", :m2 => "how goes it?", :class => :no }
+              ]
+    wrapped_dataset = FeatureSet::DataSet.wrap_dataset(dataset)
+    builder.before_build_features(wrapped_dataset)
+
+    builder.idfs.should == {
+                             :m1 => { "hello" => 2, "world" => 3 },
+                             :m2 => { "how" => 3, "goes" => 2 }
+                           }
+
+    builder.build_features(wrapped_dataset.first[:m1], :m1, wrapped_dataset.first).should == { "wv_hello" => (2/3.0), "wv_world" => (1/3.0) }
+    builder.build_features(wrapped_dataset.first[:m2], :m2, wrapped_dataset.first).should == { "wv_how" => (1/2.0), "wv_goes" => (1/2.0) }
+
+    builder.build_features(wrapped_dataset[1][:m1], :m1, wrapped_dataset[1]).should == { "wv_hello" => 0, "wv_world" => (1/2.0) }
+    builder.build_features(wrapped_dataset[1][:m2], :m2, wrapped_dataset[1]).should == { "wv_how" => (1/1.0) , "wv_goes" => 0 }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: feature_set
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-17 00:00:00.000000000Z
+date: 2012-01-23 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70136561584740 !ruby/object:Gem::Requirement
+  requirement: &70355551090860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70136561584740
+  version_requirements: *70355551090860
 - !ruby/object:Gem::Dependency
-  name: wwood-rarff
-  requirement: &70136561584320 !ruby/object:Gem::Requirement
+  name: iterationlabs-rarff
+  requirement: &70355551090340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70136561584320
+  version_requirements: *70355551090340
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70136561583900 !ruby/object:Gem::Requirement
+  requirement: &70355551089920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70136561583900
+  version_requirements: *70355551089920
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &70136561583480 !ruby/object:Gem::Requirement
+  requirement: &70355551075900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70136561583480
+  version_requirements: *70355551075900
 description: FeatureSet is a Ruby library for generating feature vectors from textual
   data.  It can output in ARFF format for experimentation with Weka.
 email:
@@ -70,17 +70,19 @@ files:
 - Rakefile
 - feature_set.gemspec
 - lib/feature_set.rb
-- lib/feature_set/builder.rb
 - lib/feature_set/data/cusswords.txt
+- lib/feature_set/data_set.rb
 - lib/feature_set/datum.rb
-- lib/feature_set/feature_builder/base.rb
-- lib/feature_set/feature_builder/cuss.rb
-- lib/feature_set/feature_builder/word_vector.rb
+- lib/feature_set/feature_builders/base.rb
+- lib/feature_set/feature_builders/cuss.rb
+- lib/feature_set/feature_builders/emoticon.rb
+- lib/feature_set/feature_builders/word_vector.rb
 - lib/feature_set/version.rb
-- spec/feature_set/builder_spec.rb
+- spec/feature_set/data_set_spec.rb
 - spec/feature_set/datum_spec.rb
-- spec/feature_set/feature/cuss_spec.rb
-- spec/feature_set/feature/word_vector_spec.rb
+- spec/feature_set/feature_builders/cuss_spec.rb
+- spec/feature_set/feature_builders/emoticon_spec.rb
+- spec/feature_set/feature_builders/word_vector_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
 homepage: https://github.com/iterationlabs/feature_set
@@ -108,9 +110,10 @@ signing_key:
 specification_version: 3
 summary: Generate feature vectors from textual data
 test_files:
-- spec/feature_set/builder_spec.rb
+- spec/feature_set/data_set_spec.rb
 - spec/feature_set/datum_spec.rb
-- spec/feature_set/feature/cuss_spec.rb
-- spec/feature_set/feature/word_vector_spec.rb
+- spec/feature_set/feature_builders/cuss_spec.rb
+- spec/feature_set/feature_builders/emoticon_spec.rb
+- spec/feature_set/feature_builders/word_vector_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb

data/lib/feature_set/builder.rb

@@ -1,97 +0,0 @@
-require 'active_support'
-require 'active_support/inflector'
-
-require "feature_set/feature_builder/word_vector"
-require "feature_set/feature_builder/cuss"
-
-require "feature_set/datum"
-
-module FeatureSet
-  class Builder
-    BUILTIN_FEATURE_BUILDERS = %w[FeatureSet::FeatureBuilder::Cuss 
-                                  FeatureSet::FeatureBuilder::WordVector].map(&:constantize)
-
-    attr_accessor :options, :feature_builders, :data, :features, :name
-
-    def initialize(options = {})
-      @options = options
-      @name = options[:name]
-      @feature_builders = []
-      @features = []
-      @data = []
-    end
-    
-    def add_data(data)
-      (@data << data).flatten!
-    end
-    
-    def clear_data
-      @data = []
-    end
-    
-    def clear_features
-      @features = []
-    end
-
-    def arff
-      relation = Rarff::Relation.new(name || 'Data')
-      keys = features.first.keys
-      instances = features.map do |row|
-        keys.map do |key|
-          value = row[key]
-          if value.is_a?(String)
-            value.gsub(/\\/, "\\\\\\\\").gsub(/"/, "\\\\\"").gsub(/'/, '\\\\\'')
-          elsif value.is_a?(Symbol)
-            value.to_s
-          else
-            value
-          end
-        end
-      end
-      relation.instances = instances
-      keys.each_with_index do |key, index|
-        relation.attributes[index].name = key.to_s
-      end
-      relation
-    end
-    
-    def generate_features(opts = {})
-      wrapped_data = self.class.wrap_dataset(data)
-      feature_builders.each {|fb| fb.before_generate_features(wrapped_data) }
-      @features = generate_features_for(wrapped_data, opts.merge(:already_wrapped => true))
-    end
-
-    def generate_features_for(data, opts = {})
-      # FYI, we explicitly do not call before_generate_features because this can be used on unknown rows for classification, and
-      # we want our feature generators to keep any cached data from the previous 'generate_features' feature building call.  This is
-      # important for Wordvector, for example, since it needs to build the idf mappings beforehand and we want them used on any new data.
-      wrapped_data = opts[:already_wrapped] ? data : self.class.wrap_dataset(data)
-      wrapped_data.map do |row|
-        output_row = {}
-        
-        row.each do |key, datum|
-          (output_row[:class] = datum) and next if key == :class
-          output_row[key] = datum.value if opts[:include_original]
-
-          feature_builders.each do |builder|
-            builder.generate_features(datum, key, row).each do |feature, value|
-              output_row["#{key}_#{feature}".to_sym] = value
-            end
-          end
-        end
-        
-        output_row
-      end
-    end
-
-    def add_feature_builders(*builders)
-      builders = BUILTIN_FEATURE_BUILDERS.map(&:new) if [:all, "all"].include?(builders.first)
-      (@feature_builders << builders).flatten!
-    end
-    alias_method :add_feature_builder, :add_feature_builders
-    
-    def self.wrap_dataset(dataset)
-      dataset.map { |row| row.inject({}) { |m, (k, v)| m[k] = (k == :class ? v : Datum.new(v)) ; m } }
-    end
-  end
-end

data/lib/feature_set/feature_builder/base.rb

@@ -1,18 +0,0 @@
-module FeatureSet
-  module FeatureBuilder
-    class Base
-      attr_accessor :options
-      
-      def initialize(options = {})
-        @options = options
-      end
-      
-      def generate_features(datum, key, row)
-        raise "Please implement 'generate_features' in your subclass of FeatureBuilder::Base."
-      end
-      
-      def before_generate_features(dataset)
-      end
-    end
-  end
-end

data/lib/feature_set/feature_builder/word_vector.rb

@@ -1,45 +0,0 @@
-require "feature_set/feature_builder/base"
-
-module FeatureSet
-  module FeatureBuilder
-    class WordVector < Base
-      attr_accessor :idfs
-
-      def initialize(options = {})
-        super
-      end
-
-      def before_generate_features(dataset)
-        @idfs = {}
-        dataset.each do |row|
-          row.each do |key, datum|
-            next if key == :class
-            if datum.value.is_a?(String)
-              idfs[key] ||= {}
-              datum.token_counts.keys.each do |token|
-                idfs[key][token] ||= 0
-                idfs[key][token] += 1
-              end
-            end
-          end
-        end
-
-        num_docs = dataset.length
-        idfs.each do |feature, freqs|
-          freqs.each do |key, value|
-            idfs[feature][key] = Math.log(num_docs / value.to_f)
-          end
-        end
-        
-        def generate_features(datum, key, row)
-          return {} unless datum.value.is_a?(String)
-          num_words = datum.tokens.length.to_f
-          idfs[key].inject({}) do |memo, (word, idf)|
-            memo[word] = ((datum.token_counts[word] || 0) / num_words) * idf
-            memo
-          end
-        end
-      end
-    end
-  end
-end

data/spec/feature_set/builder_spec.rb

@@ -1,91 +0,0 @@
-require 'spec_helper'
-
-describe FeatureSet::Builder do
-  describe "adding feature builders" do
-    it "can add all known feature builders" do
-      builder = FeatureSet::Builder.new
-      builder.add_feature_builders :all
-      builder.feature_builders.map {|i| i.class}.should include(FeatureSet::FeatureBuilder::WordVector)
-      builder.feature_builders.length.should == Dir[File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "feature_set", "feature_builder", "*.rb"))].length - 1
-    end
-    
-    it "can add individual feature builders" do
-      builder = FeatureSet::Builder.new
-      builder.add_feature_builder FeatureSet::FeatureBuilder::WordVector.new
-      builder.feature_builders.length.should == 1
-    end
-
-    it "can add arrays of feature builders" do
-      builder = FeatureSet::Builder.new
-      builder.add_feature_builders [FeatureSet::FeatureBuilder::WordVector.new, FeatureSet::FeatureBuilder::Cuss.new]
-      builder.feature_builders.length.should == 2
-    end
-  end
-  
-  describe "adding data" do
-    it "should accept mappings between one or more strings and their classifications" do
-      builder = FeatureSet::Builder.new
-      builder.add_data [ { :status => "I am happy!", :class => :happy },
-                         { :status => "I am sad." , :class => :sad } ]
-      builder.data.should == [ { :status => "I am happy!", :class => :happy },
-                               { :status => "I am sad." , :class => :sad } ]
-      builder.add_data :status => "Something", :another_feature => "Something else", :class => :awesome
-      builder.data.should == [ { :status => "I am happy!", :class => :happy },
-                               { :status => "I am sad." , :class => :sad },
-                               { :status => "Something", :another_feature => "Something else", :class => :awesome } ]
-      builder.clear_data
-      builder.data.should == []
-      builder.data = [ { :status => "I am happy!", :class => :happy },
-                       { :status => "I am sad." , :class => :sad } ]
-      builder.data.should == [ { :status => "I am happy!", :class => :happy },
-                               { :status => "I am sad." , :class => :sad } ]
-    end
-  end
-  
-  describe "generating features" do
-    before do
-      @builder = FeatureSet::Builder.new
-      @builder.add_feature_builder FeatureSet::FeatureBuilder::Cuss.new
-      @builder.add_data :status => "this is some text", :class => :awesome
-      @builder.add_data :status => "this is some shitty text", :class => :less_awesome
-    end
-    
-    it "should output a row of features for every line of data" do
-      @builder.generate_features
-      @builder.features[0].should == { :status_cuss_count => 0, :class => :awesome }
-      @builder.features[1].should == { :status_cuss_count => 1, :class => :less_awesome }
-    end
-    
-    it "should make it easy to keep the original data" do
-      @builder.generate_features(:include_original => true)
-      @builder.features[0].should == { :status => "this is some text", :status_cuss_count => 0, :class => :awesome }
-      @builder.features[1].should == { :status => "this is some shitty text", :status_cuss_count => 1, :class => :less_awesome }
-    end
-
-    it "should generate features for every string" do
-      @builder.add_data :status => "text", :foo => "more shitty text", :class => :awesome
-      @builder.generate_features
-      @builder.features[1].should == { :status_cuss_count => 1, :class => :less_awesome }
-      @builder.features[2].should == { :status_cuss_count => 0, :foo_cuss_count => 1, :class => :awesome }
-    end
-
-    it "should allow generation of features on new data while leaving the old data intact" do
-      @builder.generate_features
-      num_features = @builder.features.length
-      @builder.generate_features_for([{ :status => "is this shitty text?" }, { :status => "foo bar" }]).should == [{ :status_cuss_count => 1 }, { :status_cuss_count => 0 }]
-      @builder.features.length.should == num_features
-    end
-
-    describe "outputing an ARFF file" do
-      it "should return a rarff relation object" do
-        @builder.generate_features
-        arff = @builder.arff
-        arff.should be_a(Rarff::Relation)
-        arff.attributes.first.name.should == "status_cuss_count"
-        arff.attributes.last.name.should == "class"
-        arff.to_s.should =~ /Data/
-        arff.to_s.should =~ /status_cuss_count/
-      end
-    end
-  end
-end

data/spec/feature_set/feature/cuss_spec.rb

@@ -1,16 +0,0 @@
-require 'spec_helper'
-
-describe FeatureSet::FeatureBuilder::Cuss do
-  before do
-    @builder = FeatureSet::FeatureBuilder::Cuss.new
-  end
-
-  it "should output :cuss_count as the number of distinct cuss words found" do
-    @builder.generate_features(FeatureSet::Datum.new("this fucking shit"), nil, nil).should == { :cuss_count => 2 }
-    @builder.generate_features(FeatureSet::Datum.new("this fucking fucking fucking shit"), nil, nil).should == { :cuss_count => 2 }
-  end
-
-  it "should ignore non-string features" do
-    @builder.generate_features(FeatureSet::Datum.new(2), nil, nil).should == {}
-  end
-end

data/spec/feature_set/feature/word_vector_spec.rb

@@ -1,30 +0,0 @@
-require 'spec_helper'
-
-describe FeatureSet::FeatureBuilder::WordVector do
-  it "should output a named feature for every word in the dataset, after performing tfidf" do
-    builder = FeatureSet::FeatureBuilder::WordVector.new
-    dataset = [
-                { :m1 => "hello world.  hello!", :m2 => "how goes?", :class => :yes }, 
-                { :m1 => "foo world", :m2 => "how?", :class => :no }
-              ]
-    wrapped_dataset = FeatureSet::Builder.wrap_dataset(dataset)
-    builder.before_generate_features(wrapped_dataset)
-
-    builder.idfs.should == {
-                             :m1 => { "hello" => Math.log(2/1.0), "world" => Math.log(2/2.0), "foo" => Math.log(2/1.0) },
-                             :m2 => { "how" => Math.log(2/2.0), "goes" => Math.log(2/1.0) }
-                           }
-
-    builder.generate_features(wrapped_dataset.first[:m1], :m1, wrapped_dataset.first).should == { "hello" => (2/3.0) * Math.log(2/1.0), "world" => (1/3.0) * Math.log(2/2.0), "foo" => 0 }
-    builder.generate_features(wrapped_dataset.first[:m2], :m2, wrapped_dataset.first).should == { "how" => (1/2.0) * Math.log(2/2.0), "goes" => (1/2.0) * Math.log(2/1.0) }
-
-    builder.generate_features(wrapped_dataset.last[:m1], :m1, wrapped_dataset.last).should == { "hello" => 0, "world" => (1/2.0) * Math.log(2/2.0), "foo" => (1/2.0) * Math.log(2/1.0) }
-    builder.generate_features(wrapped_dataset.last[:m2], :m2, wrapped_dataset.last).should == { "how" => (1/1.0) * Math.log(2/2.0), "goes" => 0 }
-  end
-
-  it "should ignore non-string features" do
-    builder = FeatureSet::FeatureBuilder::WordVector.new
-    builder.before_generate_features([{ :something => FeatureSet::Datum.new(2), :class => false }, { :something => FeatureSet::Datum.new(1), :class => true }])
-    builder.generate_features(FeatureSet::Datum.new(2), :something, { :something => FeatureSet::Datum.new(2), :class => false }).should == {}
-  end
-end