feature_set 0.0.1 → 0.0.2

data/lib/feature_set/builder.rb

@@ -11,35 +11,62 @@ module FeatureSet
     BUILTIN_FEATURE_BUILDERS = %w[FeatureSet::FeatureBuilder::Cuss 
                                   FeatureSet::FeatureBuilder::WordVector].map(&:constantize)
 
-    attr_accessor :options, :feature_builders, :data, :features
+    attr_accessor :options, :feature_builders, :data, :features, :name
 
     def initialize(options = {})
       @options = options
+      @name = options[:name]
       @feature_builders = []
       @features = []
       @data = []
     end
     
     def add_data(data)
-      clear_features
       (@data << data).flatten!
     end
     
     def clear_data
       @data = []
-      clear_features
     end
     
     def clear_features
       @features = []
     end
+
+    def arff
+      relation = Rarff::Relation.new(name || 'Data')
+      keys = features.first.keys
+      instances = features.map do |row|
+        keys.map do |key|
+          value = row[key]
+          if value.is_a?(String)
+            value.gsub(/\\/, "\\\\\\\\").gsub(/"/, "\\\\\"").gsub(/'/, '\\\\\'')
+          elsif value.is_a?(Symbol)
+            value.to_s
+          else
+            value
+          end
+        end
+      end
+      relation.instances = instances
+      keys.each_with_index do |key, index|
+        relation.attributes[index].name = key.to_s
+      end
+      relation
+    end
     
     def generate_features(opts = {})
-      wrapped_data_set = self.class.wrap_dataset(data)
+      wrapped_data = self.class.wrap_dataset(data)
+      feature_builders.each {|fb| fb.before_generate_features(wrapped_data) }
+      @features = generate_features_for(wrapped_data, opts.merge(:already_wrapped => true))
+    end
 
-      feature_builders.each {|fb| fb.before_generate_features(wrapped_data_set) }
-      
-      @features = wrapped_data_set.map do |row|
+    def generate_features_for(data, opts = {})
+      # FYI, we explicitly do not call before_generate_features because this can be used on unknown rows for classification, and
+      # we want our feature generators to keep any cached data from the previous 'generate_features' feature building call.  This is
+      # important for Wordvector, for example, since it needs to build the idf mappings beforehand and we want them used on any new data.
+      wrapped_data = opts[:already_wrapped] ? data : self.class.wrap_dataset(data)
+      wrapped_data.map do |row|
         output_row = {}
         
         row.each do |key, datum|
@@ -56,7 +83,7 @@ module FeatureSet
         output_row
       end
     end
-    
+
     def add_feature_builders(*builders)
       builders = BUILTIN_FEATURE_BUILDERS.map(&:new) if [:all, "all"].include?(builders.first)
       (@feature_builders << builders).flatten!

data/lib/feature_set/version.rb

@@ -1,3 +1,3 @@
 module FeatureSet
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/feature_set/builder_spec.rb

@@ -68,5 +68,24 @@ describe FeatureSet::Builder do
       @builder.features[1].should == { :status_cuss_count => 1, :class => :less_awesome }
       @builder.features[2].should == { :status_cuss_count => 0, :foo_cuss_count => 1, :class => :awesome }
     end
+
+    it "should allow generation of features on new data while leaving the old data intact" do
+      @builder.generate_features
+      num_features = @builder.features.length
+      @builder.generate_features_for([{ :status => "is this shitty text?" }, { :status => "foo bar" }]).should == [{ :status_cuss_count => 1 }, { :status_cuss_count => 0 }]
+      @builder.features.length.should == num_features
+    end
+
+    describe "outputing an ARFF file" do
+      it "should return a rarff relation object" do
+        @builder.generate_features
+        arff = @builder.arff
+        arff.should be_a(Rarff::Relation)
+        arff.attributes.first.name.should == "status_cuss_count"
+        arff.attributes.last.name.should == "class"
+        arff.to_s.should =~ /Data/
+        arff.to_s.should =~ /status_cuss_count/
+      end
+    end
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: feature_set
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-12-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70284888584540 !ruby/object:Gem::Requirement
+  requirement: &70136561584740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70284888584540
+  version_requirements: *70136561584740
 - !ruby/object:Gem::Dependency
   name: wwood-rarff
-  requirement: &70284888584120 !ruby/object:Gem::Requirement
+  requirement: &70136561584320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70284888584120
+  version_requirements: *70136561584320
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70284888583700 !ruby/object:Gem::Requirement
+  requirement: &70136561583900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70284888583700
+  version_requirements: *70136561583900
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &70284888583280 !ruby/object:Gem::Requirement
+  requirement: &70136561583480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70284888583280
+  version_requirements: *70136561583480
 description: FeatureSet is a Ruby library for generating feature vectors from textual
   data.  It can output in ARFF format for experimentation with Weka.
 email: