typed_data 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa8bfbb2363bcf13407a8bb1fdeed0099f16a5cda9296dd77489fb2b2d7bb024
-  data.tar.gz: 7c446db80a65cc04d22584f38d53c313092432e71ba64321c22d43967a976f8a
+  metadata.gz: 483f78cdcfb329a0d14f4173fa586f64eae12c8f61dbd78492f1cd85cceadbea
+  data.tar.gz: 45173313046d5a6dc54d920ed5bee399e64fd8af2a2e710e2b5051b922fcc440
 SHA512:
-  metadata.gz: 8e783163ba498a893cacff761e9d502acf0e524e8f57790cf3500ee8e29b2310133dfacd76f017d47cf81bde4f9f0550d4bc78523e7213e7dbd80adae8864215
-  data.tar.gz: 31376d90ff56ab09b0a004b0d0bc8efaed10a6ec4d89cf8eee7c4fd2532c5f533507b96f29af365d6d413475f75d2be1a206d157ce3b19bc847ab24d2c52a17f
+  metadata.gz: 4041f615dedb5782ce40d15ae80eca87f3e168b988c2d2de4d9d07d0ba05b00bc2a6e8b14b981d65575ab9f2e38b0db50653286c3093ffaa8e7a586eb7a5e5d9
+  data.tar.gz: 04f2f8e8a86d63e8f5e06066b96e983332107f116694e059e0b06cd14ebb66827cd8bddc2e85c631298518d8387d28452edd856161b412c105f9a9d05d4151fa

data/README.md

@@ -79,36 +79,42 @@ converter.convert({
   },
 })
 #=> {"int_field"=>1,
-#    "int_or_string_field"=>{"int_value"=>nil, "string_value"=>"string"},
+#    "int_or_string_field"=>{"string_value"=>"string"},
 #    "array_field"=>[1, 2],
-#    "union_type_array_field"=>
-#     [{"int_value"=>"1", "string_value"=>nil},
-#      {"int_value"=>nil, "string_value"=>"2"}],
+#    "union_type_array_field"=>[{"int_value"=>"1"}, {"string_value"=>"2"}],
 #    "nested_map_field"=>
 #     [{"key"=>"nested_map",
 #       "value"=>
-#        [{"key"=>"key1", "value"=>{"int_value"=>"1", "string_value"=>nil}},
-#         {"key"=>"key2", "value"=>{"int_value"=>nil, "string_value"=>"2"}}]}]}
+#        [{"key"=>"key1", "value"=>{"int_value"=>"1"}},
+#         {"key"=>"key2", "value"=>{"string_value"=>"2"}}]}]}
 ```
 
 You can specify a formatter for the union type keys. For example, the formatter for tables managed by [Google BigQuery Sink Connector](https://docs.confluent.io/current/connect/kafka-connect-bigquery/index.html) is like below:
 
 ```ruby
-schema = {
-  "name" => "Record",
-  "type" => "record",
-  "fields" => [
-    {
-      "name" => "int_or_string_field",
-      "type" => ["int", "string"],
-    },
-  ],
-}
-
 converter = TypedData::Converter.new(schema)
 converter.union_type_key_formatter = ->(type) { type.split("_").first }
-converter.convert({ "int_or_string_field" => "string" })
-#=> {"int_or_string_field"=>{"int"=>nil, "string"=>"string"}}
+converter.convert({
+  "int_field" => 1,
+  "int_or_string_field" => "string",
+  "array_field" => [1, 2],
+  "union_type_array_field" => [1, "2"],
+  "nested_map_field" => {
+    "nested_map" => {
+      "key1" => 1,
+      "key2" => "2",
+    },
+  },
+})
+#=> {"int_field"=>1,
+#    "int_or_string_field"=>{"string"=>"string"},
+#    "array_field"=>[1, 2],
+#    "union_type_array_field"=>[{"int"=>"1"}, {"string"=>"2"}],
+#    "nested_map_field"=>
+#     [{"key"=>"nested_map",
+#       "value"=>
+#        [{"key"=>"key1", "value"=>{"int"=>"1"}},
+#         {"key"=>"key2", "value"=>{"string"=>"2"}}]}]}
 ```
 
 

data/lib/typed_data/converter.rb

@@ -31,9 +31,9 @@ module TypedData
         when Schema::RecordType
           converted[key] = convert_record(subtype, value)
         when Schema::UnionType
-          converted[key] = convert_union(subtype, value, as_record_field: true)
+          converted[key] = convert_union(subtype, value)
         else
-          converted[key] = subtype.coerce(value, formatter: union_type_key_formatter)
+          converted[key] = subtype.coerce(value)
         end
       end
     end
@@ -55,7 +55,7 @@ module TypedData
         when Schema::UnionType
           ret << convert_union(subtype, value)
         else
-          ret << type.coerce(value, formatter: union_type_key_formatter)
+          ret << subtype.coerce(value)
         end
       end
     end
@@ -75,16 +75,15 @@ module TypedData
         when Schema::UnionType
           value = convert_union(subtype, value)
         else
-          value = type.coerce(value, formatter: union_type_key_formatter)
+          value = subtype.coerce(value)
         end
         ret << { "key" => key, "value" => value }
       end
     end
 
     # @param type [UnionType]
-    # @param as_record_field [Boolean]
     # @param map [Object]
-    def convert_union(type, value, as_record_field: false)
+    def convert_union(type, value)
       subtype = type.find_match(value)
       case subtype
       when Schema::ArrayType
@@ -95,15 +94,18 @@ module TypedData
         converted_value = convert_record(subtype, value)
       when Schema::UnionType
         converted_value = convert_union(subtype, value)
+      when Schema::NullType
+        converted_value = nil
       else
-        return type.coerce(value, formatter: union_type_key_formatter)
+        converted_value = subtype.coerce(value).to_s
       end
 
-      if as_record_field
+      if type.nullable_single?
         converted_value
+      elsif subtype.is_a?(Schema::NullType)
+        {}
       else
-        type.default_value(union_type_key_formatter)
-          .merge!(union_type_key_formatter.call(subtype.to_s) => converted_value)
+        { union_type_key_formatter.call(subtype.to_s) => converted_value }
       end
     end
   end

data/lib/typed_data/schema.rb

@@ -5,6 +5,7 @@ require "typed_data/schema/bytes_type"
 require "typed_data/schema/enum_type"
 require "typed_data/schema/float_type"
 require "typed_data/schema/int_type"
+require "typed_data/schema/long_type"
 require "typed_data/schema/map_type"
 require "typed_data/schema/null_type"
 require "typed_data/schema/record_type"
@@ -42,14 +43,16 @@ module TypedData
             values = type["values"] || type[:values]
             MapType.new(values.is_a?(Array) ? values : [values])
           when "record"
-            RecordType.new(type["fields"] || type[:fields])
+            RecordType.new(type["name"] || type[:name], type["fields"] || type[:fields])
           else
             raise UnsupportedType, "Unknown type: #{subtype}"
           end
         when "boolean"
           BooleanType.new(type, logical_type)
-        when "int", "long"
+        when "int"
           IntType.new(type, logical_type)
+        when "long"
+          LongType.new(type, logical_type)
         when "float", "double"
           FloatType.new(type, logical_type)
         when "bytes"
@@ -72,7 +75,7 @@ module TypedData
       if (schema["type"] || schema[:type]) != "record"
         raise UnsupportedType, 'The root type must be "record"'
       end
-      @root_type = RecordType.new(schema["fields"] || schema[:fields])
+      @root_type = RecordType.new(schema["name"] || schema[:name], schema["fields"] || schema[:fields])
     end
   end
 end

data/lib/typed_data/schema/array_type.rb

@@ -15,10 +15,6 @@ module TypedData
         "array_#{@type}"
       end
 
-      def coerce(value, formatter:)
-        @type.coerce(value, formatter: formatter)
-      end
-
       def primitive?
         false
       end

data/lib/typed_data/schema/bytes_type.rb

@@ -3,7 +3,7 @@
 module TypedData
   class Schema
     class BytesType < Type
-      def coerce(value, formatter:)
+      def coerce(value)
         [value].pack("m0")
       end
 

data/lib/typed_data/schema/int_type.rb

@@ -3,6 +3,8 @@
 module TypedData
   class Schema
     class IntType < Type
+      VALUE_RANGE = -2**31 .. 2**31 - 1
+
       def to_s
         if @logical_type
           "#{@name}_#{@logical_type.gsub("-", "_")}"
@@ -11,18 +13,12 @@ module TypedData
         end
       end
 
-      def coerce(value, formatter:)
+      def coerce(value)
         case @logical_type
         when "date"
           (Date.new(1970, 1, 1) + value).to_s
         when "time-millis"
           Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%T.%3N")
-        when "time-micros"
-          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%T.%6N")
-        when "timestamp-millis"
-          Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%F %T.%3N")
-        when "timestamp-micros"
-          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%F %T.%6N")
         else
           value
         end
@@ -33,7 +29,7 @@ module TypedData
       end
 
       def match?(value)
-        value.is_a?(Integer)
+        value.is_a?(Integer) && VALUE_RANGE.cover?(value)
       end
     end
   end

data/lib/typed_data/schema/long_type.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module TypedData
+  class Schema
+    class LongType < Type
+      def to_s
+        if @logical_type
+          "#{@name}_#{@logical_type.gsub("-", "_")}"
+        else
+          @name
+        end
+      end
+
+      def coerce(value)
+        case @logical_type
+        when "time-micros"
+          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%T.%6N")
+        when "timestamp-millis"
+          Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%F %T.%3N")
+        when "timestamp-micros"
+          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%F %T.%6N")
+        else
+          value
+        end
+      end
+
+      def primitive?
+        true
+      end
+
+      def match?(value)
+        value.is_a?(Integer)
+      end
+    end
+  end
+end

data/lib/typed_data/schema/map_type.rb

@@ -12,10 +12,6 @@ module TypedData
         "map_#{@type}"
       end
 
-      def coerce(value, formatter:)
-        @type.coerce(value, formatter: formatter)
-      end
-
       def primitive?
         false
       end

data/lib/typed_data/schema/record_type.rb

@@ -4,7 +4,8 @@ module TypedData
   class Schema
     class RecordType < Type
       # @param fields [Array] an array of "fields" in an Avro schema
-      def initialize(fields)
+      def initialize(name, fields)
+        @name = name
         @field_to_type = fields.each_with_object({}) do |field, h|
           h[field["name"] || field[:name]] = Schema.build_type(field["type"] || field[:type])
         end
@@ -22,7 +23,7 @@ module TypedData
       end
 
       def match?(value)
-        value.is_a?(Hash)
+        value.is_a?(Hash) && value.all? { |k, v| @field_to_type[k]&.match?(v) }
       end
     end
   end

data/lib/typed_data/schema/type.rb

@@ -12,7 +12,7 @@ module TypedData
         @name
       end
 
-      def coerce(value, formatter:)
+      def coerce(value)
         value
       end
 

data/lib/typed_data/schema/union_type.rb

@@ -6,24 +6,14 @@ module TypedData
       # @param types [Array<String>]
       def initialize(types)
         @types = types.map(&Schema.method(:build_type))
-        @nullable_primitive = @types.size == 2 && @types.any?(&:primitive?) && @types.any? { |t| t.is_a?(NullType) }
+        @nullable_single = @types.size == 2 && @types.any? { |t| t.is_a?(NullType) }
+        @nullable_primitive = @nullable_single && @types.any?(&:primitive?)
       end
 
       def to_s
         @nullable_primitive ? @types.first.to_s : "union_#{@types.map(&:to_s).join("_")}"
       end
 
-      def coerce(value, formatter:)
-        return value if @nullable_primitive
-
-        type = find_match(value)
-        if type.is_a?(NullType)
-          default_value(formatter)
-        else
-          default_value(formatter).merge!(formatter.call(type.to_s) => type.coerce(value, formatter: formatter).to_s)
-        end
-      end
-
       def primitive?
         false
       end
@@ -36,11 +26,8 @@ module TypedData
         @types.any? { |t| t.match?(value) }
       end
 
-      def default_value(formatter)
-        @types.each_with_object({}) do |t, v|
-          next if t.is_a?(NullType)
-          v[formatter.call(t.to_s)] = t.primitive? || t.is_a?(EnumType) ? nil : []
-        end
+      def nullable_single?
+        @nullable_single
       end
     end
   end

data/lib/typed_data/version.rb

@@ -1,3 +1,3 @@
 module TypedData
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: typed_data
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - abicky
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-21 00:00:00.000000000 Z
+date: 2020-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -65,6 +65,7 @@ files:
 - lib/typed_data/schema/enum_type.rb
 - lib/typed_data/schema/float_type.rb
 - lib/typed_data/schema/int_type.rb
+- lib/typed_data/schema/long_type.rb
 - lib/typed_data/schema/map_type.rb
 - lib/typed_data/schema/null_type.rb
 - lib/typed_data/schema/record_type.rb