data_frame 0.1.8

data/spec/data_frame/core/filter_spec.rb

@@ -0,0 +1,88 @@
+require File.join(File.dirname(__FILE__), "/../../spec_helper")
+
+describe "Filter" do
+  before do
+    @labels = [:these, :are, :the, :labels]
+    @df = DataFrame.new(*@labels)
+    @df.add [1,2,3,4]
+    @df.add [5, 6, 7, 8]
+  end
+  
+  it "should be able to filter a data frame with a block using an OpenStruct for each row" do
+    @df.filter!(:open_struct) {|row| row.these == 5}
+    @df.items.should eql([[5, 6, 7, 8]])
+  end
+  
+  it "should be able to filter a data frame with a block using a Hash for each row" do
+    @df.filter!(:hash) {|row| row[:these] == 5}
+    @df.items.should eql([[5, 6, 7, 8]])
+  end
+  
+  S4 = Struct.new(:one, :two, :three, :four)
+  it "should be able to filter a data frame with a block using another class that uses the row as input" do
+    @df.filter!(S4) {|row| row.one == 5}
+    @df.items.should eql([[5, 6, 7, 8]])
+  end
+  
+  it "should be able to filter a data frame with a block using an array for each row" do
+    @df.filter! {|row| row.first == 5}
+    @df.items.should eql([[5, 6, 7, 8]])
+  end
+  
+  it "should be able to do fancy things with the row as the filter" do
+    @df.filter! {|row| row.sum > 10}
+    @df.items.should eql([[5, 6, 7, 8]])
+  end
+  
+  it "should be able to generate a new data frame with filter" do
+    new_df = @df.filter(:open_struct) {|row| row.these == 5}
+    new_df.items.should eql([[5, 6, 7, 8]])
+    @df.items.should eql([[1, 2, 3, 4], [5, 6, 7, 8]])
+  end
+  
+end
+
+context "filter_by_category" do
+  
+  before do
+    @df = DataFrame.new(:weather, :date)
+
+    (1..31).each do |i|
+      @df.add [(i % 3 == 1) ? :fair : :good, Date.parse("07/#{i}/2009")]
+    end
+
+    @d1 = Date.parse("07/15/2009")
+    @d2 = Date.parse("07/31/2009")
+
+  end
+  
+  it "should be able to filter by category" do
+    filtered = @df.filter_by_category(:weather => :good)
+    filtered.weather.uniq.should eql([:good])
+    @df.weather.uniq.should be_include(:fair)
+  end
+  
+  it "should be able to manage ranges for filter values" do
+    filtered = @df.filter_by_category(:date => (@d1..@d2))
+    filtered.date.should_not be_include(Date.parse("07/01/2009"))
+    filtered.date.should_not be_include(Date.parse("07/14/2009"))
+    filtered.date.should be_include(Date.parse("07/15/2009"))
+    filtered.date.should be_include(Date.parse("07/31/2009"))
+    @df.date.should be_include(Date.parse("07/01/2009"))
+  end
+  
+  it "should be able to take an array of values to filter with" do
+    filtered = @df.filter_by_category(:date => [@d1, @d2])
+    filtered.date.should_not be_include(Date.parse("07/01/2009"))
+    filtered.date.should be_include(Date.parse("07/15/2009"))
+    filtered.date.should be_include(Date.parse("07/31/2009"))
+  end
+  
+  it "should have a destructive version" do
+    @df.filter_by_category!(:date => [@d1, @d2])
+    @df.date.should_not be_include(Date.parse("07/01/2009"))
+    @df.date.should be_include(Date.parse("07/15/2009"))
+    @df.date.should be_include(Date.parse("07/31/2009"))
+  end
+  
+end

data/spec/data_frame/core/import_spec.rb

@@ -0,0 +1,41 @@
+require File.join(File.dirname(__FILE__), "/../../spec_helper")
+
+describe "Import" do
+  
+  before do
+    @labels = [:these, :are, :the, :labels]
+    @df = DataFrame.new(*@labels)
+  end
+  
+  it "should be able to add an item" do
+    item = [1,2,3,4]
+    @df.add_item(item)
+    @df.items.should eql([item])
+  end
+
+  it "should be able to import more than one row at a time" do
+    @df.import([[2,2,2,2],[3,3,3,3],[4,4,4,4]])
+    @df.row_labels = [:twos, :threes, :fours]
+    @df.twos.should eql([2,2,2,2])
+    @df.threes.should eql([3,3,3,3])
+    @df.fours.should eql([4,4,4,4])
+  end
+
+  it "should be able to import only one row" do
+    @df.import([2,2,2,2])
+    @df.these.should eql([2])
+  end
+
+  it "should be able to import a reference to csv" do
+    contents = %{7,5,mar,fri,86.2,26.2,94.3,5.1,8.2,51,6.7,0,0
+7,4,oct,tue,90.6,35.4,669.1,6.7,18,33,0.9,0,0
+}
+
+    @labels = [:x, :y, :month, :day, :ffmc, :dmc, :dc, :isi, :temp, :rh, :wind, :rain, :area]
+    @df = DataFrame.new(@labels)
+    @df.import(contents)
+    @df.x.should eql([7,7])
+    @df.area.should eql([0,0])
+  end
+  
+end

data/spec/data_frame/core/pre_process_spec.rb

@@ -0,0 +1,103 @@
+require File.join(File.dirname(__FILE__), "/../../spec_helper")
+
+describe "PreProcess" do
+  it "should be able to j_binary_ize! a column, taking its categories and creating a column for each" do
+    df = DataFrame.new(:observations)
+    df.add [:many]
+    df.add [:fine]
+    df.add [:things]
+    df.add [:are]
+    df.add [:available]
+    df.j_binary_ize!(:observations)
+    df.observations_many.should eql([true, false, false, false, false])
+    df.observations_fine.should eql([false, true, false, false, false])
+    df.observations_things.should eql([false, false, true, false, false])
+    df.observations_are.should eql([false, false, false, true, false])
+    df.observations_available.should eql([false, false, false, false, true])
+    df.observations.should eql([:many, :fine, :things, :are, :available])
+  end
+  
+  it "should be able to j_binary_ize! a more normal column" do
+    df = DataFrame.new(:observations)
+    df.import([1,2,3,4,5,4,3,2,1].map{|e| Array(e)})
+    df.observations.add_category(:small) {|e| e <= 3}
+    df.observations.add_category(:large) {|e| e >= 3}
+    df.j_binary_ize!(:observations)
+    df.observations_small.should eql([true, true, true, false, false, false, true, true, true])
+    df.observations_large.should eql([false, false, false, true, true, true, false, false, false])
+  end
+  
+  it "should be able to j_binary_ize with non-adjacent sets (sets that allow a value to have more than one category)" do
+    df = DataFrame.new(:observations)
+    df.import([1,2,3,4,5,4,3,2,1].map{|e| Array(e)})
+    df.observations.add_category(:small) {|e| e <= 3}
+    df.observations.add_category(:large) {|e| e >= 3}
+    df.j_binary_ize!(:observations, :allow_overlap => true)
+    df.observations_small.should eql([true, true, true, false, false, false, true, true, true])
+    df.observations_large.should eql([false, false, true, true, true, true, true, false, false])
+  end
+  
+  it "should be able to hold multiple ideas of a columns categories by resetting the category and re-running j_binary_ize" do
+    df = DataFrame.new(:observations)
+    df.import([1,2,3,4,5,4,3,2,1].map{|e| Array(e)})
+    df.observations.add_category(:small) {|e| e <= 3}
+    df.observations.add_category(:large) {|e| e >= 3}
+    df.j_binary_ize!(:observations, :allow_overlap => true)
+    df.observations.set_categories(:odd => lambda{|e| e.odd?}, :even => lambda{|e| e.even?})
+    df.j_binary_ize!(:observations)
+    df.observations_small.should eql([true, true, true, false, false, false, true, true, true])
+    df.observations_large.should eql([false, false, true, true, true, true, true, false, false])
+    df.observations.should eql([1,2,3,4,5,4,3,2,1])
+    df.observations_even.should eql([false, true, false, true, false, true, false, true, false])
+    df.observations_odd.should eql([true, false, true, false, true, false, true, false, true])
+  end
+
+  context "numericize!" do
+    
+    before do
+      @df = DataFrame.new(:observations)
+      @df.import([1,2,3,4,5,4,3,2,1].map{|e| Array(e)})
+      @df.observations.add_category(:small) {|e| e <= 3}
+      @df.observations.add_category(:large) {|e| e > 3}
+    end
+    
+    it "should be able to numericize nominal data" do
+      @df.numericize!(:observations)
+      @df.numerical_observations.should eql([[1,0],[1,0],[1,0],[0,1],[0,1],[0,1],[1,0],[1,0],[1,0]])
+    end
+    
+  end
+  
+  context "categorize!" do
+    before do
+      @df = DataFrame.new(:observations)
+      @df.import([1,2,3,4,5,4,3,2,1].map{|e| Array(e)})
+      @df.observations.add_category(0) {|e| e <= 3}
+      @df.observations.add_category(1) {|e| e > 3}
+    end
+    
+    it "should be able to replace a column with its category values" do
+      @df.categorize!(:observations)
+      @df.observations.should eql([0,0,0,1,1,1,0,0,0])
+    end
+    
+    it "should be able to replace more than one column at a time" do
+      @df.duplicate!(:observations)
+      @df.observations.add_category(0) {|e| e <= 3}
+      @df.observations.add_category(1) {|e| e > 3}
+      @df.observations1.add_category(:small) {|e| e <= 3}
+      @df.observations1.add_category(:large) {|e| e > 3}
+      @df.categorize!(:observations, :observations1)
+      @df.observations.should eql([0,0,0,1,1,1,0,0,0])
+      @df.observations1.should eql([:small,:small,:small,:large,:large,:large,:small,:small,:small])
+    end
+    
+    it "should be able to categorize a column that doesn't have a range_hash setup" do
+      @df = DataFrame.new(:observations)
+      @df.import([1,2,3,4,5,4,3,2,1].map{|e| Array(e)})
+      @df.observations.range_hash.should be_nil
+      lambda{@df.categorize!(:observations)}.should_not raise_error
+      @df.observations.should eql([1,2,3,4,5,4,3,2,1])
+    end
+  end
+end

data/spec/data_frame/core/saving_spec.rb

@@ -0,0 +1,61 @@
+require File.join(File.dirname(__FILE__), "/../../spec_helper")
+
+describe "Saving" do
+  before do
+    @df = DataFrame.new(:observations)
+    @df.import([1,2,3,4,5,4,3,2,1].map{|e| Array(e)})
+    @df.observations.add_category(:small) {|e| e <= 3}
+    @df.observations.add_category(:large) {|e| e > 3}
+    @filename = "/tmp/numericized_observations"
+  end
+  
+  after do
+    `rm -rf #{@filename}`
+  end
+
+  it "should be able to save the data frame" do
+    @df.numericize!(:observations)
+    @df.save(@filename)
+    File.read(@filename).should eql(@df.to_csv)
+  end
+
+  it "should be able to save the data frame without the header" do
+    @df.save(@filename, :include_header => false)
+    File.read(@filename).should eql(@df.to_csv(false))
+  end
+  
+  it "should be able to save off a subset" do
+    @df = DataFrame.new(:observations, :junk)
+    @df.import( [1,2,3,4,5,4,3,2,1].map{ |e| [e,e] } )
+    @df.save(@filename, :subset => :observations)
+    File.read(@filename).should eql(@df.subset_from_columns(:observations).to_csv)
+  end
+  
+  it "should be able to filter the rows" do
+    @df = DataFrame.new(:observations, :junk)
+    @df.import( [1,2,3,4,5,4,3,2,1].map{ |e| [e,e] } )
+    @df.save(@filename, :subset => :observations)
+    @df.observations.add_category(:small) {|e| e <= 3}
+    @df.observations.add_category(:large) {|e| e > 3}
+    @df.save(@filename, :filter_by_category => {:observations => :small})
+    File.read(@filename).should eql(@df.filter_by_category(:observations => :small).to_csv)
+  end
+  
+  it "should have a shortcut for subset, only" do
+    @df = DataFrame.new(:observations, :junk)
+    @df.import( [1,2,3,4,5,4,3,2,1].map{ |e| [e,e] } )
+    @df.save(@filename, :only => :observations)
+    File.read(@filename).should eql(@df.subset_from_columns(:observations).to_csv)
+  end
+  
+  it "should have a shortcut for filter_by_category, filter" do
+    @df = DataFrame.new(:observations, :junk)
+    @df.import( [1,2,3,4,5,4,3,2,1].map{ |e| [e,e] } )
+    @df.save(@filename, :subset => :observations)
+    @df.observations.add_category(:small) {|e| e <= 3}
+    @df.observations.add_category(:large) {|e| e > 3}
+    @df.save(@filename, :filter => {:observations => :small})
+    File.read(@filename).should eql(@df.filter_by_category(:observations => :small).to_csv)
+  end
+  
+end

data/spec/data_frame/core/training_spec.rb

@@ -0,0 +1,72 @@
+require File.join(File.dirname(__FILE__), "/../../spec_helper")
+
+describe "Training" do
+  before do
+    @df = DataFrame.new(:one)
+    @df.import((0...100).to_a)
+  end
+
+  it "should be able to create a proportional training set from a data frame" do
+    @df.training_set(:n => 3)
+    @df.training_set.size.should eql(3)
+    @df.training_set.all? {|e| @df.items.should be_include(e)}
+  end
+  
+  it "should use the same training set unless reset is passed to it" do
+    @df.training_set(:n => 5)
+    @df.training_set.should eql(@df.training_set)
+    old = @df.training_set
+    @df.training_set(:reset => true, :n => 5)
+    @df.training_set.should_not eql(old)
+  end
+  
+  it "should be able to create a proportional training set" do
+    @df.training_set(:proportion => 0.6)
+    @df.training_set.size.should eql(60)
+    @df.training_set(:proportion => 0.42, :reset => true)
+    @df.training_set.size.should eql(42)
+    @df.training_set(:proportion => 0, :reset => true)
+    @df.training_set.size.should eql(0)
+    @df.training_set(:proportion => 1, :reset => true)
+    @df.training_set.size.should eql(100)
+  end
+  
+  it "should not have a set size exceeding the items size" do
+    @df.training_set(:proportion => 2)
+    @df.training_set.size.should eql(100)
+    @df.training_set(:n => 200, :reset => true)
+    @df.training_set.size.should eql(100)
+  end
+  
+  it "should not have any items when the proportion is calculated below 0" do
+    @df.training_set(:proportion => -2)
+    @df.training_set.size.should eql(0)
+    @df.training_set(:n => -2, :reset => true)
+    @df.training_set.size.should eql(0)
+  end
+  
+  it "should have a default proportion of 80%" do
+    @df.training_set.size.should eql(80)
+  end
+  
+  it "should offer the test_set, all items except those in the training set" do
+    @df.test_set.should eql(@df.items.exclusive_not(@df.training_set))
+  end
+  
+  it "should reset the test set when the training set is reset" do
+    @df.training_set(:n => 2)
+    @df.test_set.size.should eql(98)
+    @df.test_set.size.should eql(98)
+    @df.training_set(:n => 1, :reset => true)
+    @df.test_set.size.should eql(99)
+  end
+  
+  it "should not reset the training set when the test set is reset" do
+    hold = @df.training_set.dup
+    @df.training_set.should eql(hold)
+    @df.test_set
+    @df.test_set(:reset => true)
+    @df.training_set.should eql(hold)
+  end
+  
+end

data/spec/data_frame/data_frame_spec.rb

@@ -0,0 +1,141 @@
+require File.join(File.dirname(__FILE__), "/../spec_helper")
+
+describe DataFrame do
+  
+  before do
+    @labels = [:these, :are, :the, :labels]
+    @df = DataFrame.new(*@labels)
+  end
+  
+  it "should initialize with labels" do
+    df = DataFrame.new(*@labels)
+    df.labels.should eql(@labels)
+  end
+  
+  it "should have an optional name" do
+    @df.name = :some_name
+    @df.name.should eql(:some_name)
+  end
+  it "should initialize with an empty items list" do
+    @df.items.should be_is_a(TransposableArray)
+    @df.items.should be_empty
+  end
+  
+  it "should use just_enumerable_stats" do
+    [1,2,3].std.should eql(1)
+    lambda{[1,2,3].cor([2,3,5])}.should_not raise_error
+  end
+
+  context "column and row operations" do
+    before do
+      @df.add_item([1,2,3,4])
+      @df.add_item([5,6,7,8])
+      @df.add_item([9,10,11,12])
+    end
+    
+    it "should have a method for every label, the column in the data frame" do
+      @df.these.should eql([1,5,9])
+    end
+    
+    it "should make columns easily computable" do
+      @df.these.std.should eql([1,5,9].std)
+    end
+
+    it "should defer unknown methods to the items in the data frame" do
+      @df[0].should eql([1,2,3,4])
+      @df << [13,14,15,16]
+      @df.last.should eql([13,14,15,16])
+      @df.map { |e| e.sum }.should eql([10,26,42,58])
+    end
+    
+    it "should allow optional row labels" do
+      @df.row_labels.should eql([])
+    end
+    
+    it "should have a setter for row labels" do
+      @df.row_labels = [:other, :things, :here]
+      @df.row_labels.should eql([:other, :things, :here])
+    end
+    
+    it "should be able to access rows by their labels" do
+      @df.row_labels = [:other, :things, :here]
+      @df.here.should eql([9,10,11,12])
+    end
+    
+    it "should make rows easily computable" do
+      @df.row_labels = [:other, :things, :here]
+      @df.here.sum.should eql(42)
+    end
+  end
+  
+  it "should be able to initialize from an array" do
+    contents = %{7,5,mar,fri,86.2,26.2,94.3,5.1,8.2,51,6.7,0,0
+7,4,oct,tue,90.6,35.4,669.1,6.7,18,33,0.9,0,0
+}
+    
+    @labels = [:x, :y, :month, :day, :ffmc, :dmc, :dc, :isi, :temp, :rh, :wind, :rain, :area]
+    @df = DataFrame.new(@labels)
+    @df.import(contents)
+    @df.labels.should eql(@labels)
+  end
+  
+  context "csv" do
+    it "should compute easily from csv" do
+      contents = %{X,Y,month,day,FFMC,DMC,DC,ISI,temp,RH,wind,rain,area
+7,5,mar,fri,86.2,26.2,94.3,5.1,8.2,51,6.7,0,0
+7,4,oct,tue,90.6,35.4,669.1,6.7,18,33,0.9,0,0
+}
+      labels = [:x, :y, :month, :day, :ffmc, :dmc, :dc, :isi, :temp, :rh, :wind, :rain, :area]
+
+      @df = DataFrame.from_csv(contents)
+      @df.labels.should eql(labels)
+      @df.x.should eql([7,7])
+      @df.area.should eql([0,0])
+    end
+    
+    it "should infer a name when importing from a file" do
+      filename = "/tmp/data_frame_spec.csv"
+      contents = %{X,Y,month,day,FFMC,DMC,DC,ISI,temp,RH,wind,rain,area
+7,5,mar,fri,86.2,26.2,94.3,5.1,8.2,51,6.7,0,0
+7,4,oct,tue,90.6,35.4,669.1,6.7,18,33,0.9,0,0
+}
+      File.open(filename, 'w') {|f| f.write contents}
+      @df = DataFrame.from_csv(filename)
+      @df.name.should eql('Data Frame Spec')
+      `rm -rf #{filename}`
+    end
+  end
+  
+  it "should offer a hash-like structure of columns" do
+    @df.add [1,2,3,4]
+    @df.add [5, 6, 7, 8]
+    @df.columns[:these].should eql([1, 5])
+    @df.columns[:are].should eql([2, 6])
+    @df.columns[:the].should eql([3, 7])
+    @df.columns[:labels].should eql([4, 8])
+  end
+  
+  it "should alias items with rows" do
+    @df.add [1,2,3,4]
+    @df.add [5, 6, 7, 8]
+    @df.rows.should eql(@df.items)
+  end
+  
+  it "should be able to export a hash" do
+    @df.add [1,2,3,4]
+    @df.add [5, 6, 7, 8]
+    hash = @df.to_hash
+    values = [[1,5],[2,6],[3,7],[4,8]]
+    hash.keys.size.should eql(@labels.size)
+    hash.keys.all? {|e| @labels.should be_include(e)}
+    hash.values.size.should eql(@labels.size)
+    hash.values.all? {|e| values.should be_include(e)}
+  end
+  
+  it "should use variables like labels" do
+    @df.labels.should eql(@labels)
+    @df.variables.should eql(@labels)
+  end
+  
+  
+end