typed_data 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa8bfbb2363bcf13407a8bb1fdeed0099f16a5cda9296dd77489fb2b2d7bb024
-  data.tar.gz: 7c446db80a65cc04d22584f38d53c313092432e71ba64321c22d43967a976f8a
+  metadata.gz: 8328e9cdeaee3bbab6b796988ff1436fc913aa8be6078b99848fe96a4d109156
+  data.tar.gz: f3b471b2a4cbc86c8a8a3b07d038c373ca7272f5417f7dd4301cd25a187d4383
 SHA512:
-  metadata.gz: 8e783163ba498a893cacff761e9d502acf0e524e8f57790cf3500ee8e29b2310133dfacd76f017d47cf81bde4f9f0550d4bc78523e7213e7dbd80adae8864215
-  data.tar.gz: 31376d90ff56ab09b0a004b0d0bc8efaed10a6ec4d89cf8eee7c4fd2532c5f533507b96f29af365d6d413475f75d2be1a206d157ce3b19bc847ab24d2c52a17f
+  metadata.gz: 15cf75777af9064e4804709b3c4aef5b3f29d867d29eefa4c93e20a5216b75c1e988809ecf985bf285178e2f1e36ff513316aeb69f179c695f196a511ce439e6
+  data.tar.gz: a00c6feb6091ed05d0e28c1afb26d386cab86aa3c110792423bd17f03c8637fb8fab500ea0e220c2c6dc8942f3b6fdc8a86f3668eb23c817d53c847de9495afe

data/README.md

@@ -1,6 +1,6 @@
 # TypedData
 
-![](https://github.com/abicky/ecsmec/workflows/test/badge.svg?branch=master)
+![](https://github.com/abicky/typed_data/workflows/CI/badge.svg?branch=master)
 
 TypedData is a library that converts hash objects managed by an Avro schema so that the objects can be loaded into BigQuery.
 
@@ -79,36 +79,42 @@ converter.convert({
   },
 })
 #=> {"int_field"=>1,
-#    "int_or_string_field"=>{"int_value"=>nil, "string_value"=>"string"},
+#    "int_or_string_field"=>{"string_value"=>"string"},
 #    "array_field"=>[1, 2],
-#    "union_type_array_field"=>
-#     [{"int_value"=>"1", "string_value"=>nil},
-#      {"int_value"=>nil, "string_value"=>"2"}],
+#    "union_type_array_field"=>[{"int_value"=>"1"}, {"string_value"=>"2"}],
 #    "nested_map_field"=>
 #     [{"key"=>"nested_map",
 #       "value"=>
-#        [{"key"=>"key1", "value"=>{"int_value"=>"1", "string_value"=>nil}},
-#         {"key"=>"key2", "value"=>{"int_value"=>nil, "string_value"=>"2"}}]}]}
+#        [{"key"=>"key1", "value"=>{"int_value"=>"1"}},
+#         {"key"=>"key2", "value"=>{"string_value"=>"2"}}]}]}
 ```
 
 You can specify a formatter for the union type keys. For example, the formatter for tables managed by [Google BigQuery Sink Connector](https://docs.confluent.io/current/connect/kafka-connect-bigquery/index.html) is like below:
 
 ```ruby
-schema = {
-  "name" => "Record",
-  "type" => "record",
-  "fields" => [
-    {
-      "name" => "int_or_string_field",
-      "type" => ["int", "string"],
-    },
-  ],
-}
-
 converter = TypedData::Converter.new(schema)
 converter.union_type_key_formatter = ->(type) { type.split("_").first }
-converter.convert({ "int_or_string_field" => "string" })
-#=> {"int_or_string_field"=>{"int"=>nil, "string"=>"string"}}
+converter.convert({
+  "int_field" => 1,
+  "int_or_string_field" => "string",
+  "array_field" => [1, 2],
+  "union_type_array_field" => [1, "2"],
+  "nested_map_field" => {
+    "nested_map" => {
+      "key1" => 1,
+      "key2" => "2",
+    },
+  },
+})
+#=> {"int_field"=>1,
+#    "int_or_string_field"=>{"string"=>"string"},
+#    "array_field"=>[1, 2],
+#    "union_type_array_field"=>[{"int"=>"1"}, {"string"=>"2"}],
+#    "nested_map_field"=>
+#     [{"key"=>"nested_map",
+#       "value"=>
+#        [{"key"=>"key1", "value"=>{"int"=>"1"}},
+#         {"key"=>"key2", "value"=>{"string"=>"2"}}]}]}
 ```
 
 
@@ -120,7 +126,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/typed_data.
+Bug reports and pull requests are welcome on GitHub at https://github.com/abicky/typed_data.
 
 
 ## License

data/lib/typed_data/converter.rb

@@ -31,9 +31,9 @@ module TypedData
         when Schema::RecordType
           converted[key] = convert_record(subtype, value)
         when Schema::UnionType
-          converted[key] = convert_union(subtype, value, as_record_field: true)
+          converted[key] = convert_union(subtype, value)
         else
-          converted[key] = subtype.coerce(value, formatter: union_type_key_formatter)
+          converted[key] = subtype.coerce(value)
         end
       end
     end
@@ -55,7 +55,7 @@ module TypedData
         when Schema::UnionType
           ret << convert_union(subtype, value)
         else
-          ret << type.coerce(value, formatter: union_type_key_formatter)
+          ret << subtype.coerce(value)
         end
       end
     end
@@ -75,16 +75,15 @@ module TypedData
         when Schema::UnionType
           value = convert_union(subtype, value)
         else
-          value = type.coerce(value, formatter: union_type_key_formatter)
+          value = subtype.coerce(value)
         end
         ret << { "key" => key, "value" => value }
       end
     end
 
     # @param type [UnionType]
-    # @param as_record_field [Boolean]
     # @param map [Object]
-    def convert_union(type, value, as_record_field: false)
+    def convert_union(type, value)
       subtype = type.find_match(value)
       case subtype
       when Schema::ArrayType
@@ -95,15 +94,18 @@ module TypedData
         converted_value = convert_record(subtype, value)
       when Schema::UnionType
         converted_value = convert_union(subtype, value)
+      when Schema::NullType
+        converted_value = nil
       else
-        return type.coerce(value, formatter: union_type_key_formatter)
+        converted_value = subtype.coerce(value)
       end
 
-      if as_record_field
+      if type.nullable_single?
         converted_value
+      elsif subtype.is_a?(Schema::NullType)
+        {}
       else
-        type.default_value(union_type_key_formatter)
-          .merge!(union_type_key_formatter.call(subtype.to_s) => converted_value)
+        { union_type_key_formatter.call(subtype.to_s) => converted_value }
       end
     end
   end

data/lib/typed_data/schema.rb

@@ -5,17 +5,16 @@ require "typed_data/schema/bytes_type"
 require "typed_data/schema/enum_type"
 require "typed_data/schema/float_type"
 require "typed_data/schema/int_type"
+require "typed_data/schema/long_type"
 require "typed_data/schema/map_type"
 require "typed_data/schema/null_type"
 require "typed_data/schema/record_type"
 require "typed_data/schema/string_type"
 require "typed_data/schema/union_type"
+require "typed_data/schema/errors"
 
 module TypedData
   class Schema
-    class UnknownField < StandardError; end
-    class UnsupportedType < StandardError; end
-
     class << self
       def build_type(type, logical_type = nil)
         type = type.first if type.is_a?(Array) && type.size == 1
@@ -42,14 +41,16 @@ module TypedData
             values = type["values"] || type[:values]
             MapType.new(values.is_a?(Array) ? values : [values])
           when "record"
-            RecordType.new(type["fields"] || type[:fields])
+            RecordType.new(type["name"] || type[:name], type["fields"] || type[:fields])
           else
             raise UnsupportedType, "Unknown type: #{subtype}"
           end
         when "boolean"
           BooleanType.new(type, logical_type)
-        when "int", "long"
+        when "int"
           IntType.new(type, logical_type)
+        when "long"
+          LongType.new(type, logical_type)
         when "float", "double"
           FloatType.new(type, logical_type)
         when "bytes"
@@ -72,7 +73,7 @@ module TypedData
       if (schema["type"] || schema[:type]) != "record"
         raise UnsupportedType, 'The root type must be "record"'
       end
-      @root_type = RecordType.new(schema["fields"] || schema[:fields])
+      @root_type = RecordType.new(schema["name"] || schema[:name], schema["fields"] || schema[:fields])
     end
   end
 end

data/lib/typed_data/schema/array_type.rb

@@ -15,10 +15,6 @@ module TypedData
         "array_#{@type}"
       end
 
-      def coerce(value, formatter:)
-        @type.coerce(value, formatter: formatter)
-      end
-
       def primitive?
         false
       end

data/lib/typed_data/schema/bytes_type.rb

@@ -3,7 +3,7 @@
 module TypedData
   class Schema
     class BytesType < Type
-      def coerce(value, formatter:)
+      def coerce(value)
         [value].pack("m0")
       end
 

data/lib/typed_data/schema/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module TypedData
+  class Schema
+    class UnknownField < StandardError; end
+    class UnsupportedType < StandardError; end
+    class InvalidValue < StandardError; end
+  end
+end

data/lib/typed_data/schema/int_type.rb

@@ -3,6 +3,9 @@
 module TypedData
   class Schema
     class IntType < Type
+      VALUE_RANGE = -2**31 .. 2**31 - 1
+      SUPPORTED_LOGICAL_TYPES = %w[date time-millis]
+
       def to_s
         if @logical_type
           "#{@name}_#{@logical_type.gsub("-", "_")}"
@@ -11,18 +14,12 @@ module TypedData
         end
       end
 
-      def coerce(value, formatter:)
+      def coerce(value)
         case @logical_type
         when "date"
           (Date.new(1970, 1, 1) + value).to_s
         when "time-millis"
           Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%T.%3N")
-        when "time-micros"
-          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%T.%6N")
-        when "timestamp-millis"
-          Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%F %T.%3N")
-        when "timestamp-micros"
-          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%F %T.%6N")
         else
           value
         end
@@ -33,7 +30,7 @@ module TypedData
       end
 
       def match?(value)
-        value.is_a?(Integer)
+        value.is_a?(Integer) && VALUE_RANGE.cover?(value)
       end
     end
   end

data/lib/typed_data/schema/long_type.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module TypedData
+  class Schema
+    class LongType < Type
+      SUPPORTED_LOGICAL_TYPES = %w[time-micros timestamp-millis timestamp-micros]
+
+      def to_s
+        if @logical_type
+          "#{@name}_#{@logical_type.gsub("-", "_")}"
+        else
+          @name
+        end
+      end
+
+      def coerce(value)
+        case @logical_type
+        when "time-micros"
+          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%T.%6N")
+        when "timestamp-millis"
+          Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%F %T.%3N")
+        when "timestamp-micros"
+          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%F %T.%6N")
+        else
+          value
+        end
+      end
+
+      def primitive?
+        true
+      end
+
+      def match?(value)
+        value.is_a?(Integer)
+      end
+    end
+  end
+end

data/lib/typed_data/schema/map_type.rb

@@ -12,10 +12,6 @@ module TypedData
         "map_#{@type}"
       end
 
-      def coerce(value, formatter:)
-        @type.coerce(value, formatter: formatter)
-      end
-
       def primitive?
         false
       end

data/lib/typed_data/schema/record_type.rb

@@ -4,7 +4,8 @@ module TypedData
   class Schema
     class RecordType < Type
       # @param fields [Array] an array of "fields" in an Avro schema
-      def initialize(fields)
+      def initialize(name, fields)
+        @name = name
         @field_to_type = fields.each_with_object({}) do |field, h|
           h[field["name"] || field[:name]] = Schema.build_type(field["type"] || field[:type])
         end
@@ -21,8 +22,12 @@ module TypedData
         end
       end
 
+      def find_match(value)
+        raise InvalidValue, %Q{the value #{value.inspect} doesn't match the type #{self}}
+      end
+
       def match?(value)
-        value.is_a?(Hash)
+        value.is_a?(Hash) && value.all? { |k, v| @field_to_type[k]&.match?(v) }
       end
     end
   end

data/lib/typed_data/schema/string_type.rb

@@ -3,6 +3,8 @@
 module TypedData
   class Schema
     class StringType < Type
+      SUPPORTED_LOGICAL_TYPES = %w[uuid]
+
       def primitive?
         true
       end

data/lib/typed_data/schema/type.rb

@@ -1,10 +1,16 @@
 # frozen_string_literal: true
+require "typed_data/schema/errors"
 
 module TypedData
   class Schema
     class Type
+      SUPPORTED_LOGICAL_TYPES = []
+
       def initialize(name, logical_type = nil)
         @name = name
+        if logical_type && !self.class::SUPPORTED_LOGICAL_TYPES.include?(logical_type)
+          raise UnsupportedType, %Q{#{name} doesn't support the logical type "#{logical_type}"}
+        end
         @logical_type = logical_type
       end
 
@@ -12,7 +18,7 @@ module TypedData
         @name
       end
 
-      def coerce(value, formatter:)
+      def coerce(value)
         value
       end
 

data/lib/typed_data/schema/union_type.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require "typed_data/schema/errors"
 
 module TypedData
   class Schema
@@ -6,41 +7,29 @@ module TypedData
       # @param types [Array<String>]
       def initialize(types)
         @types = types.map(&Schema.method(:build_type))
-        @nullable_primitive = @types.size == 2 && @types.any?(&:primitive?) && @types.any? { |t| t.is_a?(NullType) }
+        @nullable_single = @types.size == 2 && @types.any? { |t| t.is_a?(NullType) }
+        @nullable_primitive = @nullable_single && @types.any?(&:primitive?)
       end
 
       def to_s
         @nullable_primitive ? @types.first.to_s : "union_#{@types.map(&:to_s).join("_")}"
       end
 
-      def coerce(value, formatter:)
-        return value if @nullable_primitive
-
-        type = find_match(value)
-        if type.is_a?(NullType)
-          default_value(formatter)
-        else
-          default_value(formatter).merge!(formatter.call(type.to_s) => type.coerce(value, formatter: formatter).to_s)
-        end
-      end
-
       def primitive?
         false
       end
 
       def find_match(value)
-        @types.find { |t| t.match?(value) }
+        @types.find { |t| t.match?(value) } or
+          raise InvalidValue, %Q{the value #{value.inspect} doesn't match the type #{@types.map(&:to_s)}}
       end
 
       def match?(value)
         @types.any? { |t| t.match?(value) }
       end
 
-      def default_value(formatter)
-        @types.each_with_object({}) do |t, v|
-          next if t.is_a?(NullType)
-          v[formatter.call(t.to_s)] = t.primitive? || t.is_a?(EnumType) ? nil : []
-        end
+      def nullable_single?
+        @nullable_single
       end
     end
   end

data/lib/typed_data/version.rb

@@ -1,3 +1,3 @@
 module TypedData
-  VERSION = "0.1.1"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: typed_data
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - abicky
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-21 00:00:00.000000000 Z
+date: 2021-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -63,8 +63,10 @@ files:
 - lib/typed_data/schema/boolean_type.rb
 - lib/typed_data/schema/bytes_type.rb
 - lib/typed_data/schema/enum_type.rb
+- lib/typed_data/schema/errors.rb
 - lib/typed_data/schema/float_type.rb
 - lib/typed_data/schema/int_type.rb
+- lib/typed_data/schema/long_type.rb
 - lib/typed_data/schema/map_type.rb
 - lib/typed_data/schema/null_type.rb
 - lib/typed_data/schema/record_type.rb
@@ -94,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A library that converts hash objects managed by an Avro schema