data_kit 0.0.2

data/lib/data_kit/patches/rcsv.rb

@@ -0,0 +1,121 @@
+require "rcsv/rcsv"
+require "rcsv/version"
+
+require "stringio"
+
+#
+# This is a temporary monkey patch to Rcsv.parse
+#  to silence warnings in Ruby 2 about #lines being deprecated
+#
+
+class Rcsv
+  def self.parse(csv_data, options = {}, &block)
+    options[:header] ||= :use
+    raw_options = {}
+
+    raw_options[:col_sep] = options[:column_separator] && options[:column_separator][0] || ','
+    raw_options[:offset_rows] = options[:offset_rows] || 0
+    raw_options[:nostrict] = options[:nostrict]
+    raw_options[:parse_empty_fields_as] = options[:parse_empty_fields_as]
+    raw_options[:buffer_size] = options[:buffer_size] || 1024 * 1024 # 1 MiB
+
+    if csv_data.is_a?(String)
+      csv_data = StringIO.new(csv_data)
+    elsif !(csv_data.respond_to?(:lines) && csv_data.respond_to?(:read))
+      inspected_csv_data = csv_data.inspect
+      raise ParseError.new("Supplied CSV object #{inspected_csv_data[0..127]}#{inspected_csv_data.size > 128 ? '...' : ''} is neither String nor looks like IO object.")
+    end
+
+    if csv_data.respond_to?(:external_encoding)
+      raw_options[:output_encoding] = csv_data.external_encoding.to_s
+    end
+
+    initial_position = csv_data.pos
+
+    case options[:header]
+    when :use
+      header = self.raw_parse(StringIO.new(csv_data.each_line.first), raw_options).first
+      raw_options[:offset_rows] += 1
+    when :skip
+      header = (0..(csv_data.each_line.first.split(raw_options[:col_sep]).count)).to_a
+      raw_options[:offset_rows] += 1
+    when :none
+      header = (0..(csv_data.each_line.first.split(raw_options[:col_sep]).count)).to_a
+    end
+
+    raw_options[:row_as_hash] = options[:row_as_hash] # Setting after header parsing
+
+    if options[:columns]
+      only_rows = []
+      except_rows = []
+      row_defaults = []
+      column_names = []
+      row_conversions = ''
+
+      header.each do |column_header|
+        column_options = options[:columns][column_header]
+        if column_options
+          if (options[:row_as_hash])
+            column_names << (column_options[:alias] || column_header)
+          end
+
+          row_defaults << column_options[:default] || nil
+
+          only_rows << case column_options[:match]
+          when Array
+            column_options[:match]
+          when nil
+            nil
+          else
+            [column_options[:match]]
+          end
+
+          except_rows << case column_options[:not_match]
+          when Array
+            column_options[:not_match]
+          when nil
+            nil
+          else
+            [column_options[:not_match]]
+          end
+
+          row_conversions << case column_options[:type]
+          when :int
+            'i'
+          when :float
+            'f'
+          when :string
+            's'
+          when :bool
+            'b'
+          when nil
+            's' # strings by default
+          else
+            fail "Unknown column type #{column_options[:type].inspect}."
+          end
+        elsif options[:only_listed_columns]
+          column_names << nil
+          row_defaults << nil
+          only_rows << nil
+          except_rows << nil
+          row_conversions << ' '
+        else
+          column_names << column_header
+          row_defaults << nil
+          only_rows << nil
+          except_rows << nil
+          row_conversions << 's'
+        end
+      end
+
+      raw_options[:column_names] = column_names if options[:row_as_hash]
+      raw_options[:only_rows] = only_rows unless only_rows.compact.empty?
+      raw_options[:except_rows] = except_rows unless except_rows.compact.empty?
+      raw_options[:row_defaults] = row_defaults unless row_defaults.compact.empty?
+      raw_options[:row_conversions] = row_conversions
+    end
+
+    csv_data.pos = initial_position
+    return self.raw_parse(csv_data, raw_options, &block)
+  end
+end

data/lib/data_kit/version.rb

@@ -0,0 +1,3 @@
+module DataKit
+  VERSION = "0.0.2"
+end

data/lib/data_kit.rb

@@ -0,0 +1,20 @@
+require "data_kit/version"
+
+# Data I/O
+require 'data_kit/csv/parser'
+require 'data_kit/csv/analyzer'
+require 'data_kit/csv/analysis'
+require 'data_kit/csv/converter'
+
+# Data Conversion
+require 'data_kit/converters/number'
+require 'data_kit/converters/integer'
+require 'data_kit/converters/boolean'
+require 'data_kit/converters/date_time'
+
+# Datasets
+require 'data_kit/dataset/field'
+require 'data_kit/dataset/schema'
+
+# Patches / Fixes
+require 'data_kit/patches/rcsv'

data/spec/converters/boolean_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe DataKit::Converters::Boolean do
+  it "should match values" do
+    ['true', 't', 'false', 'f'].each do |testcase|
+      DataKit::Converters::Boolean.match?(testcase).should == true
+    end
+  end
+
+  it "should convert value it can match" do
+    {
+      "true" => true, "t" => true,
+      "false" => false, "f" => false
+    }.each do |testcase, result|
+      DataKit::Converters::Boolean.convert(testcase).should == result
+    end
+  end
+end

data/spec/converters/date_time_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe DataKit::Converters::DateTime do
+  it "should match a date" do
+    DataKit::Converters::DateTime.match?("1/1/00").should == true
+    DataKit::Converters::DateTime.match?("2010-01-01").should == true
+    DataKit::Converters::DateTime.match?("2010-01-01 12:00:00").should == true
+    DataKit::Converters::DateTime.match?("2000-01-01T00:00:00Z").should == true
+    DataKit::Converters::DateTime.match?("2000-01-01T00:00:00+00:00").should == true
+    DataKit::Converters::DateTime.match?("10/16/10 18:24").should == true
+    DataKit::Converters::DateTime.match?("10/16/10 1:24:15").should == true
+  end
+
+  it "should convert dates it can match" do
+    {
+      '1/1/00' => "2000-01-01 00:00:00",
+      '2010-01-01' => '2010-01-01 00:00:00',
+      '2010-01-01 12:00:00' => '2010-01-01 12:00:00',
+      '2000-01-01T00:00:00' => '2000-01-01 00:00:00',
+      '2000-01-01T00:00:00Z' => '2000-01-01 00:00:00',
+      '2000-02-01T00:00:00+00:00' => '2000-02-01 00:00:00',
+      '10/1/2012 10:27:45.000000 AM' => '2012-10-01 10:27:45',
+      '10/1/2012 1:27:45.000000 AM' => '2012-10-01 01:27:45',
+      '1/1/2012 1:27:45.000000 AM' => '2012-01-01 01:27:45',
+      "10/16/10 18:24" => '2010-10-16 18:24:00'
+    }.each do |testcase, result|
+      DataKit::Converters::DateTime.convert(testcase).strftime("%Y-%m-%d %H:%M:%S").should == result
+    end
+  end
+end

data/spec/converters/integer_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe DataKit::Converters::Integer do
+  it "should match values" do
+    ["100", "-100", "1,000", "$1,000"].each do |integer|
+      reformatted = DataKit::Converters::Integer.reformat(integer)
+      DataKit::Converters::Integer.match?(reformatted).should == true
+    end
+  end
+
+  it "should convert value it can match" do
+    {
+      "100" => 100, "-100" => -100,
+      "1,000" => 1000, "$1,000" => 1000
+    }.each do |testcase, result|
+      reformatted = DataKit::Converters::Integer.reformat(testcase)
+      DataKit::Converters::Integer.convert(reformatted).should == result
+    end
+  end
+end

data/spec/converters/number_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe DataKit::Converters::Integer do
+  it "should match values" do
+    ["100.0", "-100.5", "-1,000.00", "5.6E11", "$1,000.21"].each do |number|
+      reformatted = DataKit::Converters::Number.reformat(number)
+      DataKit::Converters::Number.match?(reformatted).should == true
+    end
+  end
+
+  it "should convert value it can match" do
+    {
+      "100.0" => 100.0, "-100.5" => -100.5,
+      "-1,000.00" => -1000.00, "5.6E11" => 5.6E11, "$1,000.21" => 1000.21
+    }.each do |testcase, result|
+      reformatted = DataKit::Converters::Number.reformat(testcase)
+      DataKit::Converters::Number.convert(reformatted).should == result
+    end
+  end
+end

data/spec/csv/analysis_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe DataKit::CSV::Analysis do  
+  it "should insert a row for analysis" do
+    analysis = DataKit::CSV::Analysis.new(['field1'])
+
+    analysis.insert 'field1', '1.0'
+    analysis.insert 'field1', '2.0'
+
+    analysis.type_list('field1').should == [:number]
+  end
+
+  it "should know the number of rows with a particular type" do
+    analysis = DataKit::CSV::Analysis.new(['field1'])
+
+    analysis.insert 'field1', '1.0'
+    analysis.insert 'field1', '2.0'
+
+    analysis.type_count('field1', :number).should == 2
+  end
+
+  it "should determine the type of a field" do
+    analysis = DataKit::CSV::Analysis.new(['field1', 'field2'])
+
+    analysis.insert 'field1', '1.0'
+    analysis.insert 'field1', '2.0'
+    analysis.insert 'field2', 'str'
+    analysis.insert 'field2', 'str2'
+
+    analysis.type?('field1').should == :number
+    analysis.type?('field2').should == :string
+  end
+
+  it "should infer a string type if there non-numeric mixed types" do
+    analysis = DataKit::CSV::Analysis.new(['field1', 'field2'])
+
+    analysis.insert 'field1', '1.0'
+    analysis.insert 'field1', '2.0'
+    analysis.insert 'field2', '2.0'
+    analysis.insert 'field2', 'str2'
+
+    analysis.type?('field1').should == :number
+    analysis.type?('field2').should == :string
+  end
+
+  it "should infer a number type if there are mixed numeric types" do
+    analysis = DataKit::CSV::Analysis.new(['field1'])
+
+    analysis.insert 'field1', '1.0'
+    analysis.insert 'field1', '20'
+    analysis.insert 'field1', nil
+
+    analysis.type?('field1').should == :number
+  end
+end

data/spec/csv/analyzer_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe DataKit::CSV::Analyzer do
+  let(:path) {
+    data_path('standard.csv')
+  }
+
+  let(:csv) {
+    DataKit::CSV::Parser.new(path)
+  }
+
+  let(:iocsv) {
+    DataKit::CSV::Parser.new(File.open(path))
+  }
+
+  it "should initialize" do
+    analyzer = DataKit::CSV::Analyzer.new(csv)
+
+    analyzer.csv.should == csv
+    analyzer.keys.should == []
+    analyzer.sample_rate.should == 0.1
+  end
+
+  it "should initialize schema with an IO" do
+    analyzer = DataKit::CSV::Analyzer.new(iocsv)
+
+    analyzer.csv.should == iocsv
+    analyzer.keys.should == []
+    analyzer.sample_rate.should == 0.1
+  end
+
+  it "should execute an analysis" do
+    analysis = DataKit::CSV::Analyzer.new(csv, :sample_rate => 0.5).execute
+
+    puts analysis.inspect
+
+    analysis.type?('id').should == :integer
+    analysis.type?('first_name').should == :string
+    analysis.type?('last_name').should == :string
+    analysis.type?('email').should == :string
+    analysis.type?('country').should == :string
+    analysis.type?('ip_address').should == :string
+    analysis.type?('amount').should == :number
+    analysis.type?('active').should == :boolean
+    analysis.type?('activated_at').should == :datetime
+    analysis.type?('address').should == :string
+
+    analysis.row_count.should == 10
+    analysis.sample_count.should be < 10
+  end
+
+  it "should calculate a sample_rate" do
+    DataKit::CSV::Analyzer.sample_rate(1024).should == 1
+    DataKit::CSV::Analyzer.sample_rate(2048 * 2048).should be < 1
+  end
+end

data/spec/csv/converter_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe DataKit::CSV::Converter do
+  let(:tmpdir) {
+    Dir.mktmpdir
+  }
+
+  let(:target) {
+    File.join(tmpdir, 'data.csv')
+  }
+
+  let(:csv) {
+    DataKit::CSV::Parser.new(data_path('standard.csv'))
+  }
+
+  it "should initialize and execute" do
+    analysis = DataKit::CSV::Analyzer.analyze(csv, :sample_rate => 1)
+    converter = DataKit::CSV::Converter.new(csv, analysis, target)
+
+    converter.execute
+
+    row_count = 0
+    CSV.open(target).each { |row| row_count += 1 }
+    row_count.should == 11
+  end
+
+  it "should convert using the convience method" do
+    analysis = DataKit::CSV::Analyzer.analyze(csv, :sample_rate => 1)
+    converter = DataKit::CSV::Converter.convert(csv, analysis, target)
+
+    row_count = 0
+    CSV.open(target).each { |row| row_count += 1 }
+    row_count.should == 11
+  end
+end

data/spec/csv/parser_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+
+describe DataKit::CSV::Parser do
+  let(:path) {
+    data_path('standard.csv')
+  }
+
+  let(:crlf_path) {
+    data_path('carriage_returns.csv')
+  }
+
+  it "should initialize" do
+    csv = DataKit::CSV::Parser.new(path)
+
+    csv.path.should == path
+  end
+
+  it "should enumerate rows with a string path" do
+    csv = DataKit::CSV::Parser.new(path)
+
+    count = 0
+    csv.each_row do |row|
+      count += 1
+    end
+
+    count.should == 10
+  end
+
+  it "should enumerate rows with an IO path" do
+    csv = DataKit::CSV::Parser.new(File.open(path))
+
+    count = 0
+    csv.each_row do |row|
+      count += 1
+    end
+
+    count.should == 10
+  end
+
+  it "should enumerate rows for lines separated by CRLF" do
+    csv = DataKit::CSV::Parser.new(File.open(crlf_path))
+
+    count = 0
+    csv.each_row do |row|
+      count += 1
+    end
+
+    count.should == 10
+  end
+end

data/spec/dataset/field_spec.rb

@@ -0,0 +1,95 @@
+require 'spec_helper'
+
+describe DataKit::Dataset::Field do
+  it "should initialize" do
+    field = DataKit::Dataset::Field.new('field_name')
+
+    field.name.should == 'field_name'
+    field.key?.should == false
+    field.type.should == :string
+  end
+
+  it "should serialize" do
+    field = DataKit::Dataset::Field.new('field_name')
+
+    field.to_hash.should == {
+      'name' => 'field_name', 'type' => 'string', 'key' => false
+    }
+  end
+
+  it "should infer nil types" do
+    DataKit::Dataset::Field.type?(nil).should == :null
+  end
+
+  it "should infer integer types" do
+    DataKit::Dataset::Field.type?("100").should == :integer
+    DataKit::Dataset::Field.type?("-100").should == :integer
+    DataKit::Dataset::Field.type?("1,000").should == :integer
+    DataKit::Dataset::Field.type?("$1,000").should == :integer
+  end
+
+  it "should infer numeric types" do
+    DataKit::Dataset::Field.type?("100.0").should == :number
+    DataKit::Dataset::Field.type?("-100.5").should == :number
+    DataKit::Dataset::Field.type?("5.6E11").should == :number
+    DataKit::Dataset::Field.type?("-1,000.0").should == :number
+    DataKit::Dataset::Field.type?("$1,000.0").should == :number
+  end
+
+  it "should infer date types" do
+    DataKit::Dataset::Field.type?("2010-01-01").should == :datetime
+
+    # Excel makes everyone sad
+    DataKit::Dataset::Field.type?("1/1/00").should == :datetime
+  end
+
+  it "should infer date/time types" do
+    DataKit::Dataset::Field.type?("2010-01-01 12:00:00").should == :datetime
+  end
+
+  it "should infer boolean types" do
+    DataKit::Dataset::Field.type?("true").should == :boolean
+    DataKit::Dataset::Field.type?("false").should == :boolean
+  end
+
+  it "should infer string types" do
+    DataKit::Dataset::Field.type?("true5").should == :string
+    DataKit::Dataset::Field.type?("my string").should == :string
+  end
+
+  it "should convert nil values" do
+    DataKit::Dataset::Field.convert(nil, :string).should == nil
+  end
+
+  it "should convert integer values" do
+    DataKit::Dataset::Field.convert("100", :integer).should == 100
+    DataKit::Dataset::Field.convert("-100", :integer).should == -100
+    DataKit::Dataset::Field.convert("1,000", :integer).should == 1_000
+    DataKit::Dataset::Field.convert("$1,000", :integer).should == 1_000
+  end
+
+  it "should convert numeric values" do
+    DataKit::Dataset::Field.convert("100.0", :number).should == 100.0
+    DataKit::Dataset::Field.convert("-100.0", :number).should == -100.0
+    DataKit::Dataset::Field.convert("-1,000.0", :number).should == -1_000.0
+    DataKit::Dataset::Field.convert("5E5", :number).should == 500000.0
+    DataKit::Dataset::Field.convert("$1,000.0", :number).should == 1000.0
+  end
+
+  it "should convert boolean values" do
+    DataKit::Dataset::Field.convert("true", :boolean).should == true
+    DataKit::Dataset::Field.convert("false", :boolean).should == false
+  end
+
+  it "should convert date values" do
+    DataKit::Dataset::Field.convert("2010-01-01", :datetime).strftime("%Y-%m-%d %H:%M:%S").should == '2010-01-01 00:00:00'
+  end
+
+  it "should convert date/time values" do
+    DataKit::Dataset::Field.convert("2010-01-01 12:00:00", :datetime).strftime("%Y-%m-%d %H:%M:%S").should == '2010-01-01 12:00:00'
+  end
+
+  it "should convert string values" do
+    DataKit::Dataset::Field.convert(500, :string).should == "500"
+  end
+end

data/spec/dataset/schema_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe DataKit::Dataset::Schema do  
+  it "should initialize" do
+    schema = DataKit::Dataset::Schema.new
+    schema.fields.should == []
+  end
+
+  it "should return a list of keys" do
+    schema = DataKit::Dataset::Schema.new
+    schema.fields << DataKit::Dataset::Field.new('field')
+    schema.fields << DataKit::Dataset::Field.new('field2', key: true)
+
+    schema.keys.length.should == 1
+  end
+
+  it "should serialize to yaml" do
+    schema = DataKit::Dataset::Schema.new
+    schema.fields << DataKit::Dataset::Field.new('field')
+    schema.to_yaml.should == schema.fields.collect(&:to_hash).to_yaml
+  end
+end

data/spec/fixtures/carriage_returns.csv

@@ -0,0 +1 @@
+draft_order,player,position,avg_draft_position,avg_bid_value

data/spec/fixtures/standard.csv

@@ -0,0 +1,11 @@
+id,first_name,last_name,email,country,ip_address,amount,active,activated_at,address
+1,Todd,Hamilton,thamilton@plambee.edu,Norfolk Island,66.157.128.241,$7.72,true,10/3/1955,0582 Dwight Street
+2,Melissa,Kelly,mkelly@twinte.name,Singapore,204.221.167.233,$4.33,false,7/9/2013,06 Cardinal Crossing
+3,Donald,Wheeler,dwheeler@edgeify.mil,Madagascar,34.201.104.193,$2.92,true,12/12/1993,4 Del Sol Hill
+4,Ruby,Hall,rhall@cogilith.com,USSR,237.243.109.67,$8.27,false,12/15/1975,7 Ramsey Avenue
+5,Jessica,Cole,jcole@shuffletag.info,Cyprus,25.40.138.137,$8.16,false,6/2/1939,8142 Novick Hill
+6,Doris,Nelson,dnelson@zoombox.biz,Svalbard and Jan Mayen Islands,233.43.155.229,$6.26,false,5/23/1948,596 Veith Road
+7,Robert,Hansen,rhansen@miboo.edu,Ghana,41.194.33.211,$4.90,true,9/28/1999,529 Oak Pass
+8,Matthew,Freeman,mfreeman@midel.name,Sudan,53.186.162.65,$6.63,false,5/24/1996,70682 Declaration Center
+9,Julia,Nelson,jnelson@skajo.net,Vatican City State (Holy See),249.49.124.178,$9.80,true,1/26/1940,96 Hermina Lane
+10,Wanda,Palmer,wpalmer@ntags.biz,Indonesia,0.200.163.200,$5.89,false,1/1/1959,26837 Donald Trail

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+require "codeclimate-test-reporter"
+CodeClimate::TestReporter.start
+
+require 'rubygems'
+require 'bundler/setup'
+
+SimpleCov.start do
+  add_filter "/spec"
+end
+
+require 'tmpdir'
+require 'data_kit'
+
+RSpec.configure do |config|
+  def data_path(file)
+    File.join(File.dirname(__FILE__), 'fixtures', file)
+  end
+end