schema-inference 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e783c172078cf4000dc5efabd8d5ad85cc3072ff
-  data.tar.gz: b84bdc0aa7c1cfaec9e6228f70c60e070fffe30f
+  metadata.gz: 2f26792c5eb829a7d9971d33fa1f8f5969cde64b
+  data.tar.gz: 810e9c35acc28b06c5d2858d92e3d2fb08cbabb5
 SHA512:
-  metadata.gz: 9157d2db16ed2b70a6d731f11d363e976a914635dffe32fe1a750576890e3cca675ef844bb34c48ffce19696d7fc7d3826f3e82696a0c5549d5d63176359d950
-  data.tar.gz: b97153fe64406c10783f28d90e94e3f66a246138c4abeb2bb002cb359f807a211215ca16f6068d74fc74346c3a801eabb354c21c9f42271dc4c48fadc262845b
+  metadata.gz: 2c8e5baefeb83e102dffc70d7f7a57638f5e1d9c886701947c032f93e1f6b1cc82fb370dcd8873b9e414c41fd291fb089b0ccb73383fa5764e18cd8fc2118935
+  data.tar.gz: 162cace85faea7adc7e66861b874e1ea13a9b01b761ba735e615c4f5d1bc95a2e426754381de87e9e1b38c4776e881bc84e7c1984dee68a2cec8bebc10d6f123

data/lib/schema/inference/schema_inferrer.rb

@@ -50,30 +50,53 @@ module Schema
         raise ArgumentError, 'dataset must be an array or a hash'
       end
 
+      FIXNUM_MAX = (2**(0.size * 8 -2) -1)
+
       def data_schema(data)
         table_schema = {}
         data.each do |record|
           # fetch the record schema & update the general schema
           rec_schema = record_schema(record)
-          rec_schema.each do |field_schema|
-            table_schema[field_schema[:field]] ||= {type: field_schema[:type], usage_count: 0}
-            if table_schema[field_schema[:field]][:type] != field_schema[:type]
-              if table_schema[field_schema[:field]][:type] == NilClass
-                table_schema[field_schema[:field]][:type] = field_schema[:type]
-              elsif field_schema[:type] != nil
-                table_schema[field_schema[:field]][:type] = lowest_common_type(field_schema[:type], table_schema[field_schema[:field]][:type])
+          rec_schema.each do |field|
+            field_schema = table_schema[field[:field]] ||= {type: field[:type], usage_count: 0}
+            field_schema = table_schema[field[:field]]
+            if field_schema[:type] != field[:type]
+              if field_schema[:type] == NilClass
+                # if it was set as nil, we now set it to a concrete type
+                field_schema[:type] = field[:type]
+              elsif field[:type] != nil
+                # if it had a different (non-nil) type, then try to upcast
+                field_schema[:type] = lowest_common_type(field[:type], field_schema[:type])
               end
             end
-            table_schema[field_schema[:field]][:usage_count] += 1
-            table_schema[field_schema[:field]][:types] ||= {}
-            table_schema[field_schema[:field]][:types][field_schema[:type]] ||= 0
-            table_schema[field_schema[:field]][:types][field_schema[:type]] += 1
+
+            field_schema[:usage_count] += 1
+            field_schema[:types] ||= {}
+            field_schema[:types][field[:type]] ||= { count: 0 }
+            field_schema[:types][field[:type]][:count] += 1
+
+            if type_has_min_max?(field[:type])
+              field_size = value_length(field[:inferred_value])
+              field_schema[:types][field[:type]][:min] = [field_schema[:types][field[:type]][:min] || FIXNUM_MAX, field_size].min
+              field_schema[:types][field[:type]][:max] = [field_schema[:types][field[:type]][:max] || 0, field_size].max
+            end
+
+
           end
         end
 
         table_schema
       end
 
+      def type_has_min_max?(type)
+        type == String || NumericTypes.include?(type)
+      end
+
+      def value_length(value)
+        return value.length if value.is_a?(String)
+        value # leave as-is otherwise
+      end
+
       def process_schema_results(results, total_count, extended)
         # aggregate the results
         table_schema = results[0]
@@ -81,10 +104,14 @@ module Schema
           table_schema.each { |k, v|
             next if res[k].blank?
 
-            # aggregate types count
-            res[k][:types].each { |type, count|
-              table_schema[k][:types][type] ||= 0
-              table_schema[k][:types][type] += count
+            # aggregate types count, set min and max
+            res[k][:types].each { |type, info|
+              table_schema[k][:types][type] ||= { count: 0 }
+              table_schema[k][:types][type][:count] += info[:count]
+              if type_has_min_max?(type)
+                table_schema[k][:types][type][:min] = [table_schema[k][:types][type][:min] || FIXNUM_MAX, info[:min]].min
+                table_schema[k][:types][type][:max] = [table_schema[k][:types][type][:max] || 0, info[:max]].max
+              end
             }
 
             # aggregate other informations
@@ -174,7 +201,7 @@ module Schema
             record_schema(x, field_name)
           }
         else
-          { field: name, type: detect_type_of(record) }
+          { field: name, type: detect_type_of(record), inferred_value: inferred_value_of(record) }
         end
       end
 
@@ -196,6 +223,20 @@ module Schema
         Object
       end
 
+      def inferred_value_of(value)
+        return value unless value.is_a?(String)
+
+        return value.to_i if value =~ /^[-+]?[0-9]+$/
+        return value.to_f if value =~ /^[-+]?[0-9]*\.?[0-9]+$/
+        return true  if value.downcase == 'true'
+        return false if value.downcase == 'false'
+
+        time_value = Timeliness.parse(value)
+        return time_value if time_value
+
+        value
+      end
+
       def key_access_tokens(key:)
         key.split(separator).map { |token|
           # only parse integers for array indexing

data/lib/schema/inference/version.rb

@@ -1,5 +1,5 @@
 module Schema
   module Inference
-    VERSION = '1.1.0'
+    VERSION = '1.2.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: schema-inference
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-20 00:00:00.000000000 Z
+date: 2016-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler