data_kit 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd32e91aed4ac35a8dbb104e577d9d7454025733
-  data.tar.gz: 31e8c8ef04ba50e94bffb8a29104362a14e56e08
+  metadata.gz: e93d042b5d65309bf8d99ff311b0a9d9dd3b2899
+  data.tar.gz: 7ddf9b7d5a11ffc59ef0c093dcd2378270099410
 SHA512:
-  metadata.gz: 77effe4f14b155f04c3f7d177a4ba0072cb02d558a2916bb66d8709484422e28e34153090b467a828dfc3ebf14d967d97f0b66cf5ba9a6c9ccbca14cc62952ab
-  data.tar.gz: e94f0b6657c0b7a105a6dd324df4ce8824c21d6acce11eee50a55973ef108873641e340597ef989cedda71043a10466e28f05540fa8a23ad6cb2176c8e2a002e
+  metadata.gz: f4231fed6d650d0b5ba39771469c014ccfe70f1f360c9a299604ee5a4e616566b5bc7cf8feab9cbb5f89e3ae4bfe5387cf528dfa664a9f0d25151a6bade4f34b
+  data.tar.gz: 5caa3516fcb3b8aa5eeef7ef57e9a8c27e9108a580ebd77917502e2c4f9bd1d2f2c90459a85663028e081684a8b3dab7845cdda8879f267fe9a2a966e0fe975e

data/lib/data_kit/csv/schema_analysis.rb

@@ -6,12 +6,24 @@ module DataKit
       attr_reader :row_count
       attr_reader :sample_count
 
-      def initialize(fields)
+      attr_reader :type_hints
+      attr_reader :use_type_hints
+
+      def initialize(fields, options = {})
         @fields, @types = fields, {}
         @row_count, @sample_count = 0, 0
 
+        @type_hints = {}
+
+        if options[:use_type_hints].nil? || options[:use_type_hints] == false
+          @use_type_hints = false
+        else
+          @use_type_hints = true
+        end
+
         fields.each do |field_name|
           @types[field_name] = {}
+          @type_hints[field_name] = :string
           Dataset::Field::Types.each do |type|
             @types[field_name][type] = 0
           end
@@ -27,7 +39,14 @@ module DataKit
       end
 
       def insert(field_name, value)
-        @types[field_name][Dataset::Field.type?(value)] += 1
+        if use_type_hints
+          type = Dataset::Field.type?(value, type_hints[field_name])
+          @type_hints[field_name] = type # cache the most recent type
+        else
+          type = Dataset::Field.type?(value)
+        end
+
+        @types[field_name][type] += 1
       end
 
       def field_types

data/lib/data_kit/dataset/field.rb

@@ -23,8 +23,13 @@ module DataKit
       end
 
       class << self
-        def type?(value)
+        def type?(value, hint_type = nil)
           return :null if value.nil?
+
+          if hint_type && is_type?(value, hint_type)
+            return hint_type
+          end
+
           reformatted = Converters::Number.reformat(value)
 
           if Converters::Integer.match?(reformatted)
@@ -40,6 +45,16 @@ module DataKit
           end
         end
 
+        def is_type?(value, type)
+          case type
+          when :integer  then Converters::Integer.match?(Converters::Number.reformat(value))
+          when :number   then Converters::Number.match?(Converters::Number.reformat(value))
+          when :boolean  then Converters::Boolean.match?(value)
+          when :datetime then Converters::DateTime.match?(value)
+          when :string   then false
+          end
+        end
+
         def convert(value, type)
           return nil if type == :null || value.nil?
           reformatted = Converters::Number.reformat(value)

data/lib/data_kit/version.rb

@@ -1,3 +1,3 @@
 module DataKit
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_kit
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Mode Analytics
@@ -198,3 +198,4 @@ test_files:
 - spec/fixtures/standard.csv
 - spec/fixtures/utf8.csv
 - spec/spec_helper.rb
+has_rdoc: