data_kit 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a0057bb3688ec21f40b1c696675839e8408b98f
-  data.tar.gz: 24429670b09afe543ff132f45bc44f14a5840d5e
+  metadata.gz: 30b22c22f9efa81ca1011684ed418acfd1fce0e1
+  data.tar.gz: 0875d12a8b029661945098eebe35965420a5dbb8
 SHA512:
-  metadata.gz: 55c82ae032798b11107b6fed0bc69b1c83b2503c56dde2274c5b02f4f57f82391f92184548048f834bc3beb0658467cfedb1af436accdbd4ae1c66a9b2f79480
-  data.tar.gz: 4b89e5f6031c28a4a3403e291e92ca278d470368ae437c740155e84f5e4acc11865204b5eb91d228b99855983c82b9eb663e29267853e092e3341d1897e55746
+  metadata.gz: 6520bb22fbce4620c012a1757b84582d68e577ccb2d8f730017f4afa9ac8190f9813a056582e220c012a5c0f0ec206fd4abc40af36810b23125758a2cfaa3180
+  data.tar.gz: 9518d32e467cce05341449ce4409ccad339e680f38bf8b65ad357d8f96a2311110d7eb3d5f2c826af56bbae5fed0baea16674837bbab8656be548926fab7a040

data/data_kit.gemspec

@@ -19,7 +19,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Runtime Dependencies
-  spec.add_runtime_dependency "rcsv"
   spec.add_runtime_dependency "timeliness"
 
   # Development Dependencies

data/lib/data_kit.rb

@@ -20,7 +20,4 @@ require 'data_kit/converters/date_time'
 
 # Datasets
 require 'data_kit/dataset/field'
-require 'data_kit/dataset/schema'
-
-# Patches / Fixes
-require 'data_kit/patches/rcsv'
+require 'data_kit/dataset/schema'

data/lib/data_kit/csv/converter.rb

@@ -16,11 +16,15 @@ module DataKit
 
       def execute
         ::CSV.open(output_path, 'wb') do |writer|
-          writer << csv.headers
+          first = true
+          converted = []
           csv.each_row do |row|
-            writer << csv.headers.collect do |field_name|
-              convert(row[field_name], field_types[field_name])
+            if first
+              first = false
+              writer << csv.headers
             end
+            
+            writer << convert_row(csv.headers, row)
           end
         end
       end
@@ -39,7 +43,15 @@ module DataKit
 
       private
 
-      def convert(value, type)
+      def convert_row(headers, row)
+        converted = []
+        headers.each_with_index do |field_name, index|
+          converted << convert_value(row[index], field_types[field_name])
+        end
+        converted
+      end
+
+      def convert_value(value, type)
         if value.nil? || type == :string || type == :empty
           return value.to_s
         else

data/lib/data_kit/csv/field_analyzer.rb

@@ -14,16 +14,21 @@ module DataKit
       end
 
       def execute
+        first = true
+        analysis = nil
         random = Random.new
 
-        field_name = csv.headers[field_pos]
-        analysis = FieldAnalysis.new(field_name, { :match_type => match_type })
-
         csv.each_row do |row|
+          if first
+            first = false
+            field_name = csv.headers[field_pos]
+            analysis = FieldAnalysis.new(field_name, { :match_type => match_type })
+          end
+
           analysis.increment_total
           if random.rand <= sampling_rate
             analysis.increment_sample
-            analysis.insert(row[field_name])
+            analysis.insert(row[field_pos])
           end
         end
 

data/lib/data_kit/csv/parser.rb

@@ -1,4 +1,4 @@
-require 'rcsv'
+require 'csv'
 
 module DataKit
   module CSV
@@ -9,29 +9,25 @@ module DataKit
 
       def initialize(path)
         @path = path
-        
-        set_handle
-        set_headers
+        set_handle      
       end
 
       def each_row(&block)
+        first = true
         handle.rewind
-        Rcsv.parse(handle, :header => :skip, :columns => columns, :row_as_hash => true) do |row|
-          yield row
+        
+        ::CSV.parse(handle, converters: nil) do |row|
+          if first == true
+            first = false
+            @headers = row
+          else
+            yield row
+          end
         end
       end
 
       private
 
-      def columns
-        index = -1
-        @columns ||= headers.inject({}) do |result, field_name|
-          index += 1
-          result[index] = { :alias => field_name }
-          result
-        end
-      end
-
       def set_handle
         if path.is_a?(IO)
           @handle = path
@@ -44,11 +40,6 @@ module DataKit
           {:invalid => :replace, :undef => :replace, :replace => ''}
         )
       end
-
-      def set_headers
-        handle.rewind
-        Rcsv.parse(handle, :header => :none) { |row| @headers = row; break }
-      end
     end
   end
 end

data/lib/data_kit/csv/schema_analyzer.rb

@@ -19,15 +19,21 @@ module DataKit
       end
 
       def execute
+        first = true
+        analysis = nil
         random = Random.new
-        analysis = SchemaAnalysis.new(csv.headers, :use_type_hints => use_type_hints)
 
         csv.each_row do |row|
+          if first
+            first = false
+            analysis = SchemaAnalysis.new(csv.headers, :use_type_hints => use_type_hints)
+          end
+
           analysis.increment_total
           if random.rand <= sampling_rate
             analysis.increment_sample
-            row.keys.each do |field_name|
-              analysis.insert(field_name.to_s, row[field_name])
+            row.each_with_index do |value, index|
+              analysis.insert(csv.headers[index].to_s, value)
             end
           end
         end

data/lib/data_kit/version.rb

@@ -1,3 +1,3 @@
 module DataKit
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: data_kit
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Mode Analytics
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-18 00:00:00.000000000 Z
+date: 2014-01-22 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: rcsv
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: timeliness
   requirement: !ruby/object:Gem::Requirement
@@ -137,7 +123,6 @@ files:
 - lib/data_kit/csv/schema_analyzer.rb
 - lib/data_kit/dataset/field.rb
 - lib/data_kit/dataset/schema.rb
-- lib/data_kit/patches/rcsv.rb
 - lib/data_kit/version.rb
 - spec/converters/boolean_spec.rb
 - spec/converters/date_time_spec.rb

data/lib/data_kit/patches/rcsv.rb

@@ -1,124 +0,0 @@
-require "rcsv/rcsv"
-require "rcsv/version"
-
-require "stringio"
-
-#
-# This is a temporary monkey patch to Rcsv.parse
-#  to silence warnings in Ruby 2 about #lines being deprecated
-#
-
-class Rcsv
-  def self.parse(csv_data, options = {}, &block)
-    options[:header] ||= :use
-    raw_options = {}
-
-    raw_options[:col_sep] = options[:column_separator] && options[:column_separator][0] || ','
-    raw_options[:offset_rows] = options[:offset_rows] || 0
-    raw_options[:nostrict] = options[:nostrict]
-    raw_options[:parse_empty_fields_as] = options[:parse_empty_fields_as]
-    raw_options[:buffer_size] = options[:buffer_size] || 1024 * 1024 # 1 MiB
-
-    if csv_data.is_a?(String)
-      csv_data = StringIO.new(csv_data)
-    elsif !(csv_data.respond_to?(:lines) && csv_data.respond_to?(:read))
-      inspected_csv_data = csv_data.inspect
-      raise ParseError.new("Supplied CSV object #{inspected_csv_data[0..127]}#{inspected_csv_data.size > 128 ? '...' : ''} is neither String nor looks like IO object.")
-    end
-
-    if csv_data.respond_to?(:external_encoding)
-      raw_options[:output_encoding] = csv_data.external_encoding.to_s
-    end
-
-    initial_position = csv_data.pos
-
-    first_line = csv_data.each_line.first
-    field_count = first_line.split(raw_options[:col_sep]).length
-
-    case options[:header]
-    when :use
-      header = self.raw_parse(StringIO.new(first_line), raw_options).first
-      raw_options[:offset_rows] += 1
-    when :skip
-      header = (0..field_count).to_a
-      raw_options[:offset_rows] += 1
-    when :none
-      header = (0..field_count).to_a
-    end
-
-    raw_options[:row_as_hash] = options[:row_as_hash] # Setting after header parsing
-
-    if options[:columns]
-      only_rows = []
-      except_rows = []
-      row_defaults = []
-      column_names = []
-      row_conversions = ''
-
-      header.each do |column_header|
-        column_options = options[:columns][column_header]
-        if column_options
-          if (options[:row_as_hash])
-            column_names << (column_options[:alias] || column_header)
-          end
-
-          row_defaults << column_options[:default] || nil
-
-          only_rows << case column_options[:match]
-          when Array
-            column_options[:match]
-          when nil
-            nil
-          else
-            [column_options[:match]]
-          end
-
-          except_rows << case column_options[:not_match]
-          when Array
-            column_options[:not_match]
-          when nil
-            nil
-          else
-            [column_options[:not_match]]
-          end
-
-          row_conversions << case column_options[:type]
-          when :int
-            'i'
-          when :float
-            'f'
-          when :string
-            's'
-          when :bool
-            'b'
-          when nil
-            's' # strings by default
-          else
-            fail "Unknown column type #{column_options[:type].inspect}."
-          end
-        elsif options[:only_listed_columns]
-          column_names << nil
-          row_defaults << nil
-          only_rows << nil
-          except_rows << nil
-          row_conversions << ' '
-        else
-          column_names << column_header
-          row_defaults << nil
-          only_rows << nil
-          except_rows << nil
-          row_conversions << 's'
-        end
-      end
-
-      raw_options[:column_names] = column_names if options[:row_as_hash]
-      raw_options[:only_rows] = only_rows unless only_rows.compact.empty?
-      raw_options[:except_rows] = except_rows unless except_rows.compact.empty?
-      raw_options[:row_defaults] = row_defaults unless row_defaults.compact.empty?
-      raw_options[:row_conversions] = row_conversions
-    end
-
-    csv_data.pos = initial_position
-    return self.raw_parse(csv_data, raw_options, &block)
-  end
-end