typed_data 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 483f78cdcfb329a0d14f4173fa586f64eae12c8f61dbd78492f1cd85cceadbea
-  data.tar.gz: 45173313046d5a6dc54d920ed5bee399e64fd8af2a2e710e2b5051b922fcc440
+  metadata.gz: 72c692bdd403124256b1670dbaf3fde4a626272779ec7727e07c52e6ca6f4afd
+  data.tar.gz: 9fb2db3eeed81eb8141a01ca7f3b9aa3fc791b5d3790c6903fe63886567f9dc9
 SHA512:
-  metadata.gz: 4041f615dedb5782ce40d15ae80eca87f3e168b988c2d2de4d9d07d0ba05b00bc2a6e8b14b981d65575ab9f2e38b0db50653286c3093ffaa8e7a586eb7a5e5d9
-  data.tar.gz: 04f2f8e8a86d63e8f5e06066b96e983332107f116694e059e0b06cd14ebb66827cd8bddc2e85c631298518d8387d28452edd856161b412c105f9a9d05d4151fa
+  metadata.gz: e881d11c4543afa34e40d7ccdb40c8f302ac2bb5bf55a29e367e3960cfc885412089dc6b6c40d4167621727141d2f3bd123d5a82d3e6f4ff8b915a7fae65b6b7
+  data.tar.gz: 82b34a7ee6180ea0f12c2a065c5b971d55b495cf15d89b865d92bf0b344b5e6c4065a5993d87a35ed92f95810f27194f6fc6be460b1bb312efb7720d0a2b66a3

data/README.md

@@ -1,6 +1,6 @@
 # TypedData
 
-![](https://github.com/abicky/ecsmec/workflows/test/badge.svg?branch=master)
+![](https://github.com/abicky/typed_data/workflows/CI/badge.svg?branch=master)
 
 TypedData is a library that converts hash objects managed by an Avro schema so that the objects can be loaded into BigQuery.
 
@@ -23,6 +23,8 @@ Or install it yourself as:
 
 ## Usage
 
+### Use as Ruby library
+
 ```ruby
 require "typed_data"
 
@@ -81,19 +83,20 @@ converter.convert({
 #=> {"int_field"=>1,
 #    "int_or_string_field"=>{"string_value"=>"string"},
 #    "array_field"=>[1, 2],
-#    "union_type_array_field"=>[{"int_value"=>"1"}, {"string_value"=>"2"}],
+#    "union_type_array_field"=>[{"int_value"=>1}, {"string_value"=>"2"}],
 #    "nested_map_field"=>
 #     [{"key"=>"nested_map",
 #       "value"=>
-#        [{"key"=>"key1", "value"=>{"int_value"=>"1"}},
+#        [{"key"=>"key1", "value"=>{"int_value"=>1}},
 #         {"key"=>"key2", "value"=>{"string_value"=>"2"}}]}]}
 ```
 
-You can specify a formatter for the union type keys. For example, the formatter for tables managed by [Google BigQuery Sink Connector](https://docs.confluent.io/current/connect/kafka-connect-bigquery/index.html) is like below:
+You can specify the formatter for union type keys. The default formatter is `:bigquery`, which is used for BigQuery tables created by loading Avro data for the first time.
+The other formatter is `:avro`, the formatter for the Avro JSON encoding, which is used in tables managed by [Google BigQuery Sink Connector](https://docs.confluent.io/current/connect/kafka-connect-bigquery/index.html):
+
 
 ```ruby
-converter = TypedData::Converter.new(schema)
-converter.union_type_key_formatter = ->(type) { type.split("_").first }
+converter = TypedData::Converter.new(schema, key_formatter: :avro)
 converter.convert({
   "int_field" => 1,
   "int_or_string_field" => "string",
@@ -109,14 +112,92 @@ converter.convert({
 #=> {"int_field"=>1,
 #    "int_or_string_field"=>{"string"=>"string"},
 #    "array_field"=>[1, 2],
-#    "union_type_array_field"=>[{"int"=>"1"}, {"string"=>"2"}],
+#    "union_type_array_field"=>[{"int"=>1}, {"string"=>"2"}],
 #    "nested_map_field"=>
 #     [{"key"=>"nested_map",
 #       "value"=>
-#        [{"key"=>"key1", "value"=>{"int"=>"1"}},
+#        [{"key"=>"key1", "value"=>{"int"=>1}},
 #         {"key"=>"key2", "value"=>{"string"=>"2"}}]}]}
 ```
 
+`TypedData::Restorer` enables you to restore the converted data:
+
+```ruby
+restorer = TypedData::Restorer.new(schema)
+restorer.restore({
+  "int_field" => 1,
+  "int_or_string_field" => { "string_value" => "string" },
+  "array_field" => [1, 2],
+  "union_type_array_field" => [
+    { "int_value" => 1 },
+    { "string_value" => "2" },
+  ],
+  "nested_map_field" => [
+    {
+      "key" => "nested_map",
+      "value" =>[
+        {
+          "key" => "key1",
+          "value" => { "int_value" => 1 }
+        },
+        {
+          "key" => "key2",
+          "value" => { "string_value" => "2"}
+        },
+      ],
+    },
+  ],
+})
+#=> {"int_field"=>1,
+#    "int_or_string_field"=>"string",
+#    "array_field"=>[1, 2],
+#    "union_type_array_field"=>[1, "2"],
+#    "nested_map_field"=>{"nested_map"=>{"key1"=>1, "key2"=>"2"}}}
+```
+
+### Use as CLI
+
+```
+$ typed-data help
+Commands:
+  typed-data convert [file] --schema=SCHEMA  # Convert data in an encoding similar to Avro JSON encoding
+  typed-data help [COMMAND]                  # Describe available commands or one specific command
+  typed-data restore [file] --schema=SCHEMA  # Restore converted data
+
+$ typed-data help convert
+Usage:
+  typed-data convert [file] --schema=SCHEMA
+
+Options:
+  --schema=SCHEMA        # Path to Avro schema file
+  [--key-format=FORMAT]  # Format for union type key
+                         # Default: bigquery
+                         # Possible values: bigquery, avro
+
+Description:
+  This command converts data in an encoding similar to Avro JSON encoding. You can specify the file in
+  JSON format or JSON Lines format. If the file option is ommited, the command read data from stdin.
+$ typed-data help restore
+Usage:
+  typed-data restore [file] --schema=SCHEMA
+
+Options:
+  --schema=SCHEMA        # Path to Avro schema file
+  [--key-format=FORMAT]  # Format for union type key
+                         # Default: bigquery
+                         # Possible values: bigquery, avro
+
+Description:
+  This command restores converted data. You can specify the file in JSON format or JSON Lines format. If
+  the file option is ommited, the command read data from stdin.
+```
+
+For example, you can restore the data loaded into a BigQuery table like below:
+
+```
+$ bq query --format json 'SELECT * FROM <table>' | typed-data restore --schema /path/to/avsc
+```
+
 
 ## Development
 
@@ -126,7 +207,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/typed_data.
+Bug reports and pull requests are welcome on GitHub at https://github.com/abicky/typed_data.
 
 
 ## License

data/example/convert.rb

@@ -0,0 +1,22 @@
+require "json"
+require "typed_data"
+
+schema = JSON.parse(File.read(File.join(__dir__, "schema.avsc")))
+data = JSON.parse(File.read(File.join(__dir__, "data.jsonl")))
+
+puts "Schema:"
+pp schema
+puts
+
+puts "Input data:"
+pp data
+puts
+
+converter = TypedData::Converter.new(schema)
+puts "Converted data with the default key formatter:"
+pp converter.convert(data)
+puts
+
+converter = TypedData::Converter.new(schema, key_formatter: :avro)
+puts "Converted data with the key formatter :avro:"
+pp converter.convert(data)

data/example/convert.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+data=data.jsonl
+
+echo "Input data:"
+cat $data
+echo
+
+cmd="typed-data convert --schema schema.avsc $data"
+echo "Execute: $cmd"
+eval $cmd
+echo
+
+cmd="cat $data | typed-data convert --schema schema.avsc"
+echo "Execute: $cmd"
+eval $cmd

data/example/converted_data.jsonl

@@ -0,0 +1 @@
+{"int_field":1,"int_or_string_field":{"string_value":"string"},"array_field":[1,2],"union_type_array_field":[{"int_value":1},{"string_value":"2"}],"nested_map_field":[{"key":"nested_map","value":[{"key":"key1","value":{"int_value":1}},{"key":"key2","value":{"string_value":"2"}}]}]}

data/example/data.jsonl

@@ -0,0 +1 @@
+{"int_field":1,"int_or_string_field":"string","array_field":[1,2],"union_type_array_field":[1,"2"],"nested_map_field":{"nested_map":{"key1":1,"key2":"2"}}}

data/example/restore.rb

@@ -0,0 +1,17 @@
+require "json"
+require "typed_data"
+
+schema = JSON.parse(File.read(File.join(__dir__, "schema.avsc")))
+data = JSON.parse(File.read(File.join(__dir__, "converted_data.jsonl")))
+
+puts "Schema:"
+pp schema
+puts
+
+puts "Input data:"
+pp data
+puts
+
+restorer = TypedData::Restorer.new(schema)
+puts "Restored data:"
+pp restorer.restore(data)

data/example/restore.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cd $(dirname $0)
+
+data=converted_data.jsonl
+
+echo "Input data:"
+cat $data
+echo
+
+cmd="typed-data restore --schema schema.avsc $data"
+echo "Execute: $cmd"
+eval $cmd
+echo
+
+cmd="cat $data | typed-data restore --schema schema.avsc"
+echo "Execute: $cmd"
+eval $cmd

data/example/schema.avsc

@@ -0,0 +1,47 @@
+{
+  "name": "Record",
+  "type": "record",
+  "fields": [
+    {
+      "name": "int_field",
+      "type": "int"
+    },
+    {
+      "name": "int_or_string_field",
+      "type": [
+        "int",
+        "string"
+      ]
+    },
+    {
+      "name": "array_field",
+      "type": {
+        "type": "array",
+        "items": "int"
+      }
+    },
+    {
+      "name": "union_type_array_field",
+      "type": {
+        "type": "array",
+        "items": [
+          "int",
+          "string"
+        ]
+      }
+    },
+    {
+      "name": "nested_map_field",
+      "type": {
+        "type": "map",
+        "values": {
+          "type": "map",
+          "values": [
+            "int",
+            "string"
+          ]
+        }
+      }
+    }
+  ]
+}

data/exe/typed-data

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require "typed_data/cli"
+
+TypedData::CLI.start(ARGV)

data/lib/typed_data.rb

@@ -1,7 +1,6 @@
 require "typed_data/converter"
+require "typed_data/restorer"
 require "typed_data/version"
 
 module TypedData
-  class Error < StandardError; end
-  # Your code goes here...
 end

data/lib/typed_data/cli.rb

@@ -0,0 +1,68 @@
+require "json"
+require "thor"
+require "typed_data/converter"
+require "typed_data/restorer"
+
+module TypedData
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    desc "convert [file]", "Convert data in an encoding similar to Avro JSON encoding"
+    long_desc <<~DESC
+      This command converts data in an encoding similar to Avro JSON encoding.
+      You can specify the file in JSON format or JSON Lines format.
+      If the file option is ommited, the command read data from stdin.
+    DESC
+    option :schema, desc: "Path to Avro schema file", required: true
+    option :"key-format", desc: "Format for union type key", enum: %w[bigquery avro], default: "bigquery", banner: "FORMAT"
+    def convert(file = nil)
+      process(TypedData::Converter, :convert, file)
+    end
+
+    desc "restore [file]", "Restore converted data"
+    long_desc <<~DESC
+      This command restores converted data.
+      You can specify the file in JSON format or JSON Lines format.
+      If the file option is ommited, the command read data from stdin.
+    DESC
+    option :schema, desc: "Path to Avro schema file", required: true
+    option :"key-format", desc: "Format for union type key", enum: %w[bigquery avro], default: "bigquery", banner: "FORMAT"
+    def restore(file = nil)
+      process(TypedData::Restorer, :restore, file)
+    end
+
+    private
+
+    def process(processor_class, method_name, file)
+      abort_if_not_exist(options[:schema])
+      abort_if_not_exist(file) if file
+
+      schema = JSON.parse(File.read(options[:schema]))
+      processor = processor_class.new(schema, key_formatter: options[:"key-format"].to_sym)
+
+      input = file ? File.open(file) : $stdin
+      first_line = input.readline.lstrip
+      if first_line.start_with?("[")
+        first_line << input.read
+        JSON.parse(first_line).each do |record|
+          puts processor.public_send(method_name, record).to_json
+        end
+      else
+        records = input
+        puts processor.public_send(method_name, JSON.parse(first_line)).to_json
+        input.each do |line|
+          puts processor.public_send(method_name, JSON.parse(line)).to_json
+        end
+      end
+    end
+
+    def abort_if_not_exist(file)
+      unless File.exist?(file)
+        $stderr.puts("#{file} doesn't exit")
+        exit(1)
+      end
+    end
+  end
+end

data/lib/typed_data/converter.rb

@@ -1,111 +1,116 @@
 # frozen_string_literal: true
+require "typed_data/key_formatter"
 require "typed_data/schema"
 
 module TypedData
   class Converter
-    attr_accessor :union_type_key_formatter
-
     # @param schema [Hash] an Avro schema
-    def initialize(schema)
+    # @param key_formatter [Symbol]
+    def initialize(schema, key_formatter: :bigquery)
       @schema = Schema.new(schema)
-      @union_type_key_formatter = ->(type) { "#{type}_value" }
+      @union_type_key_formatter = KeyFormatter.find(key_formatter)
+    end
+
+    def union_type_key_formatter=(formatter)
+      warn "DEPRECATION WARNING: #{__method__} is deprecated. Specify the key_formatter :avsc to TypedData::Converter.new instead."
+      @union_type_key_formatter = formatter
     end
 
     # @param data [Hash]
     def convert(data)
-      convert_record(@schema.root_type, data)
+      @schema.root_type.accept(self, data)
+    end
+
+    # @param type [TypedData::Schema::Type]
+    # @param value [Object]
+    def visit(type, value)
+      value
     end
 
-    private
+    # @param type [TypedData::Schema::BytesType]
+    # @param value [String]
+    def visit_bytes(type, value)
+      [value].pack("m0")
+    end
+
+    # @param type [TypedData::Schema::IntType]
+    # @param logical_type [String, nil] a logical type of the int type
+    # @param value [Integer]
+    def visit_int(type, logical_type, value)
+      case logical_type
+      when "date"
+        (Date.new(1970, 1, 1) + value).to_s
+      when "time-millis"
+        Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%T.%3N")
+      else
+        value
+      end
+    end
 
-    # @param type [RecordType]
+    # @param type [TypedData::Schema::LongType]
+    # @param logical_type [String, nil] logical type of the long type
+    # @param value [Integer]
+    def visit_long(type, logical_type, value)
+      case logical_type
+      when "time-micros"
+        Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%T.%6N")
+      when "timestamp-millis"
+        Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%F %T.%3N")
+      when "timestamp-micros"
+        Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%F %T.%6N")
+      else
+        value
+      end
+    end
+
+    # @param type [TypedData::Schema::RecordType]
     # @param record [Hash{String => Object}]
-    def convert_record(type, record)
+    def visit_record(type, record)
       record.each_with_object({}) do |(key, value), converted|
-        subtype = type.find_type(key)
-        case subtype
-        when Schema::ArrayType
-          converted[key] = convert_array(subtype, value)
-        when Schema::MapType
-          converted[key] = convert_map(subtype, value)
-        when Schema::RecordType
-          converted[key] = convert_record(subtype, value)
-        when Schema::UnionType
-          converted[key] = convert_union(subtype, value)
-        else
-          converted[key] = subtype.coerce(value)
-        end
+        converted[key] = type.find_type(key).accept(self, value)
       end
     end
 
-    # @param type [ArrayType]
+    # @param type [TypedData::Schema::ArrayType]
     # @param array [Array<Object>]
-    def convert_array(type, array)
+    def visit_array(type, array)
       array.each_with_object([]) do |value, ret|
         next if value.nil?
 
-        subtype = type.find_match(value)
-        case subtype
-        when Schema::ArrayType
-          ret.concat(convert_array(subtype, value))
-        when Schema::MapType
-          ret << convert_map(subtype, value)
-        when Schema::RecordType
-          ret << convert_record(subtype, value)
-        when Schema::UnionType
-          ret << convert_union(subtype, value)
+        converted_value = type.element_type.accept(self, value)
+        if type.element_type.is_a?(Schema::ArrayType)
+          # BigQuery doesn't support nested arrays
+          ret.concat(converted_value)
         else
-          ret << subtype.coerce(value)
+          ret << converted_value
         end
       end
     end
 
-    # @param type [MapType]
+    # @param type [TypedData::Schema::MapType]
     # @param map [Hash{String => Object}]
-    def convert_map(type, map)
+    def visit_map(type, map)
       map.each_with_object([]) do |(key, value), ret|
-        subtype = type.find_match(value)
-        case subtype
-        when Schema::ArrayType
-          value = convert_array(subtype, value)
-        when Schema::MapType
-          value = convert_map(subtype, value)
-        when Schema::RecordType
-          value = convert_record(subtype, value)
-        when Schema::UnionType
-          value = convert_union(subtype, value)
-        else
-          value = subtype.coerce(value)
-        end
-        ret << { "key" => key, "value" => value }
+        ret << { "key" => key, "value" => type.element_type.accept(self, value) }
       end
     end
 
-    # @param type [UnionType]
+    # @param type [TypedData::Schema::UnionType]
+    # @param types [Array<TypedData::Schema::Type>] types the union type includes
     # @param map [Object]
-    def convert_union(type, value)
-      subtype = type.find_match(value)
-      case subtype
-      when Schema::ArrayType
-        converted_value = convert_array(subtype, value)
-      when Schema::MapType
-        converted_value = convert_map(subtype, value)
-      when Schema::RecordType
-        converted_value = convert_record(subtype, value)
-      when Schema::UnionType
-        converted_value = convert_union(subtype, value)
-      when Schema::NullType
-        converted_value = nil
-      else
-        converted_value = subtype.coerce(value).to_s
+    def visit_union(type, types, value)
+      element_type = types.find { |t| t.match?(value) }
+      if element_type.nil?
+        raise Schema::InvalidValue, %Q{the value #{value.inspect} doesn't match the type #{types.map(&:to_s)}}
       end
+      converted_value = element_type.accept(self, value)
 
       if type.nullable_single?
         converted_value
-      elsif subtype.is_a?(Schema::NullType)
+      elsif element_type.is_a?(Schema::NullType)
         {}
       else
-        { union_type_key_formatter.call(subtype.to_s) => converted_value }
+        { @union_type_key_formatter.call(element_type.to_s) => converted_value }
       end
     end
   end

data/lib/typed_data/key_formatter.rb

@@ -0,0 +1,17 @@
+module TypedData
+  class KeyFormatter
+    class UnknownFormatter < StandardError; end
+
+    UNION_TYPE_KEY_FORMATTERS = {
+      bigquery: ->(type) { "#{type}_value" },
+      avro: ->(type) { type.split("_").first },
+    }
+
+    # @param formatter [Symbol]
+    def self.find(formatter)
+      UNION_TYPE_KEY_FORMATTERS.fetch(formatter) do
+        raise UnknownFormatter, "Unknown formatter: #{formatter}"
+      end
+    end
+  end
+end

data/lib/typed_data/restorer.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+require "time"
+require "typed_data/key_formatter"
+require "typed_data/schema"
+
+module TypedData
+  class Restorer
+    # @param schema [Hash] an Avro schema
+    # @param key_formatter [Symbol]
+    def initialize(schema, key_formatter: :bigquery)
+      @schema = Schema.new(schema)
+      @union_type_key_formatter = KeyFormatter.find(key_formatter)
+    end
+
+    # @param data [Hash]
+    def restore(data)
+      @schema.root_type.accept(self, data)
+    end
+
+    # @param type [TypedData::Schema::Type]
+    # @param value [Object]
+    def visit(type, value)
+      value
+    end
+
+    # @param type [TypedData::Schema::BytesType]
+    # @param value [String]
+    def visit_bytes(type, value)
+      value.unpack("m0").first
+    end
+
+    # @param type [TypedData::Schema::IntType]
+    # @param logical_type [String, nil] a logical type of the int type
+    # @param value [Integer]
+    def visit_int(type, logical_type, value)
+      case logical_type
+      when "date"
+        (Date.parse(value) - Date.new(1970, 1, 1)).to_i
+      when "time-millis"
+        t = Time.parse(value)
+        (t.sec + t.min * 60 + t.hour * 60**2) * 10**3 + t.nsec / 10**6
+      else
+        value
+      end
+    end
+
+    # @param type [TypedData::Schema::LongType]
+    # @param logical_type [String, nil] logical type of the long type
+    # @param value [Integer]
+    def visit_long(type, logical_type, value)
+      case logical_type
+      when "time-micros"
+        t = Time.parse(value)
+        (t.sec + t.min * 60 + t.hour * 60**2) * 10**6 + t.nsec / 10**3
+      when "timestamp-millis"
+        t = parse_as_utc(value)
+        t.to_i * 10**3 + t.nsec / 10**6
+      when "timestamp-micros"
+        t = parse_as_utc(value)
+        t.to_i * 10**6 + t.nsec / 10**3
+      else
+        value
+      end
+    end
+
+    # @param type [TypedData::Schema::RecordType]
+    # @param record [Hash{String => Object}]
+    def visit_record(type, record)
+      record.each_with_object({}) do |(key, value), restored|
+        restored[key] = type.find_type(key).accept(self, value)
+      end
+    end
+
+    # @param type [TypedData::Schema::ArrayType]
+    # @param array [Array<Object>]
+    def visit_array(type, array)
+      array.each_with_object([]) do |value, ret|
+        next if value.nil?
+
+        if type.element_type.is_a?(Schema::ArrayType)
+          # BigQuery doesn't support nested arrays
+          ret << type.element_type.element_type.accept(self, value)
+        else
+          ret << type.element_type.accept(self, value)
+        end
+      end
+    end
+
+    # @param type [TypedData::Schema::MapType]
+    # @param map [Hash{String => Object}]
+    def visit_map(type, array)
+      array.each_with_object({}) do |hash, ret|
+        ret[hash["key"]] = type.element_type.accept(self, hash["value"])
+      end
+    end
+
+    # @param type [TypedData::Schema::UnionType]
+    # @param types [Array<TypedData::Schema::Type>] types the union type includes
+    # @param map [Object]
+    def visit_union(type, types, value)
+      if type.nullable_single?
+        return if value.nil?
+
+        element_type = types.find { |t| !t.is_a?(Schema::NullType) }
+        element_type.accept(self, value)
+      else
+        value_without_nil = value.compact
+        return if value_without_nil.empty?
+
+        k = value_without_nil.keys.first
+        v = value_without_nil.values.first
+        element_type = types.find { |t| k == @union_type_key_formatter.call(t.to_s) }
+        element_type.accept(self, v)
+      end
+    end
+
+    private
+
+    # @param time [String]
+    def parse_as_utc(time)
+      d = Date._parse(time)
+      Time.utc(d[:year], d[:mon], d[:mday], d[:hour], d[:min], d[:sec], d.fetch(:sec_fraction, 0) * 1000000)
+    end
+  end
+end

data/lib/typed_data/schema.rb

@@ -11,13 +11,13 @@ require "typed_data/schema/null_type"
 require "typed_data/schema/record_type"
 require "typed_data/schema/string_type"
 require "typed_data/schema/union_type"
+require "typed_data/schema/errors"
 
 module TypedData
   class Schema
-    class UnknownField < StandardError; end
-    class UnsupportedType < StandardError; end
-
     class << self
+      # @param type [String, Hash{Symbol => Object}, Array<Hash{Symbol => Object}>]
+      # @param logical_type [String, nil]
       def build_type(type, logical_type = nil)
         type = type.first if type.is_a?(Array) && type.size == 1
 
@@ -25,27 +25,26 @@ module TypedData
         when Array
           UnionType.new(type)
         when Hash
-          subtype = type["type"] || type[:type]
-          logical_type = type["logicalType"] || type[:logicalType]
-          if logical_type
-            return build_type(subtype, logical_type)
+          actual_type = type[:type]
+          if type[:logicalType]
+            return build_type(actual_type, type[:logicalType])
           end
 
-          case subtype
+          case actual_type
           when "enum"
-            EnumType.new(type["name"] || type[:name], type["symbols"] || type[:symbols])
+            EnumType.new(type[:name], type[:symbols])
           when "fixed"
-            BytesType.new(type["name"] || type[:name] || "bytes")
+            BytesType.new(type[:name] || "bytes")
           when "array"
-            items = type["items"] || type[:items]
+            items = type[:items]
             ArrayType.new(items.is_a?(Array) ? items : [items])
           when "map"
-            values = type["values"] || type[:values]
+            values = type[:values]
             MapType.new(values.is_a?(Array) ? values : [values])
           when "record"
-            RecordType.new(type["name"] || type[:name], type["fields"] || type[:fields])
+            RecordType.new(type[:name], type[:fields])
           else
-            raise UnsupportedType, "Unknown type: #{subtype}"
+            raise UnsupportedType, "Unknown type: #{actual_type}"
           end
         when "boolean"
           BooleanType.new(type, logical_type)
@@ -71,11 +70,25 @@ module TypedData
 
     # @param schema [Hash] an Avro schema
     def initialize(schema)
-      @schema = schema
-      if (schema["type"] || schema[:type]) != "record"
-        raise UnsupportedType, 'The root type must be "record"'
+      @schema = deep_symbolize_keys(schema)
+      @root_type = Schema.build_type(@schema)
+    end
+
+    private
+
+    # @param hash [Object]
+    # @return [Object] an object with symbolized keys
+    def deep_symbolize_keys(o)
+      case o
+      when Array
+        o.map(&method(:deep_symbolize_keys))
+      when Hash
+        o.each_with_object({}) do |(k, v), h|
+          h[k.to_sym] = deep_symbolize_keys(v)
+        end
+      else
+        o
       end
-      @root_type = RecordType.new(schema["name"] || schema[:name], schema["fields"] || schema[:fields])
     end
   end
 end

data/lib/typed_data/schema/array_type.rb

@@ -4,27 +4,27 @@ require "typed_data/schema/type"
 module TypedData
   class Schema
     class ArrayType < Type
-      attr_reader :fields
+      attr_reader :element_type
 
       # @param types [Array<String>]
       def initialize(types)
-        @type = Schema.build_type(types.select { |t| t != "null" })
+        @element_type = Schema.build_type(types)
+      end
+
+      def accept(visitor, value)
+        visitor.visit_array(self, value)
       end
 
       def to_s
-        "array_#{@type}"
+        "array_#{@element_type}"
       end
 
       def primitive?
         false
       end
 
-      def find_match(value)
-        @type.match?(value) ? @type : @type.find_match(value)
-      end
-
       def match?(value)
-        value.is_a?(Array) && value.all? { |v| @type.match?(v) }
+        value.is_a?(Array) && value.all? { |v| @element_type.match?(v) }
       end
     end
   end

data/lib/typed_data/schema/bytes_type.rb

@@ -3,8 +3,8 @@
 module TypedData
   class Schema
     class BytesType < Type
-      def coerce(value)
-        [value].pack("m0")
+      def accept(visitor, value)
+        visitor.visit_bytes(self, value)
       end
 
       def primitive?

data/lib/typed_data/schema/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module TypedData
+  class Schema
+    class UnknownField < StandardError; end
+    class UnsupportedType < StandardError; end
+    class InvalidValue < StandardError; end
+  end
+end

data/lib/typed_data/schema/int_type.rb

@@ -4,6 +4,11 @@ module TypedData
   class Schema
     class IntType < Type
       VALUE_RANGE = -2**31 .. 2**31 - 1
+      SUPPORTED_LOGICAL_TYPES = %w[date time-millis]
+
+      def accept(visitor, value)
+        visitor.visit_int(self, @logical_type, value)
+      end
 
       def to_s
         if @logical_type
@@ -13,17 +18,6 @@ module TypedData
         end
       end
 
-      def coerce(value)
-        case @logical_type
-        when "date"
-          (Date.new(1970, 1, 1) + value).to_s
-        when "time-millis"
-          Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%T.%3N")
-        else
-          value
-        end
-      end
-
       def primitive?
         true
       end

data/lib/typed_data/schema/long_type.rb

@@ -3,6 +3,12 @@
 module TypedData
   class Schema
     class LongType < Type
+      SUPPORTED_LOGICAL_TYPES = %w[time-micros timestamp-millis timestamp-micros]
+
+      def accept(visitor, value)
+        visitor.visit_long(self, @logical_type, value)
+      end
+
       def to_s
         if @logical_type
           "#{@name}_#{@logical_type.gsub("-", "_")}"
@@ -11,19 +17,6 @@ module TypedData
         end
       end
 
-      def coerce(value)
-        case @logical_type
-        when "time-micros"
-          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%T.%6N")
-        when "timestamp-millis"
-          Time.at(value / 1_000, value % 1_000 * 1_000).utc.strftime("%F %T.%3N")
-        when "timestamp-micros"
-          Time.at(value / 1_000_000, value % 1_000_000).utc.strftime("%F %T.%6N")
-        else
-          value
-        end
-      end
-
       def primitive?
         true
       end

data/lib/typed_data/schema/map_type.rb

@@ -3,25 +3,27 @@
 module TypedData
   class Schema
     class MapType < Type
+      attr_reader :element_type
+
       # @param types [Array<String>]
       def initialize(types)
-        @type = Schema.build_type(types)
+        @element_type = Schema.build_type(types)
+      end
+
+      def accept(visitor, value)
+        visitor.visit_map(self, value)
       end
 
       def to_s
-        "map_#{@type}"
+        "map_#{@element_type}"
       end
 
       def primitive?
         false
       end
 
-      def find_match(value)
-        @type.match?(value) ? @type : @type.find_match(value)
-      end
-
       def match?(value)
-        value.is_a?(Hash) && value.all? { |_, v| @type.match?(v) }
+        value.is_a?(Hash) && value.all? { |_, v| @element_type.match?(v) }
       end
     end
   end

data/lib/typed_data/schema/record_type.rb

@@ -3,14 +3,19 @@
 module TypedData
   class Schema
     class RecordType < Type
+      # @param name [String]
       # @param fields [Array] an array of "fields" in an Avro schema
       def initialize(name, fields)
         @name = name
         @field_to_type = fields.each_with_object({}) do |field, h|
-          h[field["name"] || field[:name]] = Schema.build_type(field["type"] || field[:type])
+          h[field[:name]] = Schema.build_type(field[:type])
         end
       end
 
+      def accept(visitor, value)
+        visitor.visit_record(self, value)
+      end
+
       def primitive?
         false
       end

data/lib/typed_data/schema/string_type.rb

@@ -3,6 +3,8 @@
 module TypedData
   class Schema
     class StringType < Type
+      SUPPORTED_LOGICAL_TYPES = %w[uuid]
+
       def primitive?
         true
       end

data/lib/typed_data/schema/type.rb

@@ -1,19 +1,25 @@
 # frozen_string_literal: true
+require "typed_data/schema/errors"
 
 module TypedData
   class Schema
     class Type
+      SUPPORTED_LOGICAL_TYPES = []
+
       def initialize(name, logical_type = nil)
         @name = name
+        if logical_type && !self.class::SUPPORTED_LOGICAL_TYPES.include?(logical_type)
+          raise UnsupportedType, %Q{#{name} doesn't support the logical type "#{logical_type}"}
+        end
         @logical_type = logical_type
       end
 
-      def to_s
-        @name
+      def accept(visitor, value)
+        visitor.visit(self, value)
       end
 
-      def coerce(value)
-        value
+      def to_s
+        @name
       end
 
       def primitive?

data/lib/typed_data/schema/union_type.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require "typed_data/schema/errors"
 
 module TypedData
   class Schema
@@ -7,21 +8,21 @@ module TypedData
       def initialize(types)
         @types = types.map(&Schema.method(:build_type))
         @nullable_single = @types.size == 2 && @types.any? { |t| t.is_a?(NullType) }
-        @nullable_primitive = @nullable_single && @types.any?(&:primitive?)
+        @nullable_primitive_type = @types.find(&:primitive?) if @nullable_single
+      end
+
+      def accept(visitor, value)
+        visitor.visit_union(self, @types, value)
       end
 
       def to_s
-        @nullable_primitive ? @types.first.to_s : "union_#{@types.map(&:to_s).join("_")}"
+        @nullable_primitive_type&.to_s || "union_#{@types.map(&:to_s).join("_")}"
       end
 
       def primitive?
         false
       end
 
-      def find_match(value)
-        @types.find { |t| t.match?(value) }
-      end
-
       def match?(value)
         @types.any? { |t| t.match?(value) }
       end

data/lib/typed_data/version.rb

@@ -1,3 +1,3 @@
 module TypedData
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/typed_data.gemspec

@@ -22,6 +22,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "thor"
+
   spec.add_development_dependency "avro"
   spec.add_development_dependency "google-cloud-bigquery"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: typed_data
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - abicky
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-25 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: avro
   requirement: !ruby/object:Gem::Requirement
@@ -42,7 +56,8 @@ description: TypedData is a library that converts hash objects managed by an Avr
   schema so that the objects can be loaded into BigQuery.
 email:
 - takeshi.arabiki@gmail.com
-executables: []
+executables:
+- typed-data
 extensions: []
 extra_rdoc_files: []
 files:
@@ -56,13 +71,25 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- example/convert.rb
+- example/convert.sh
+- example/converted_data.jsonl
+- example/data.jsonl
+- example/restore.rb
+- example/restore.sh
+- example/schema.avsc
+- exe/typed-data
 - lib/typed_data.rb
+- lib/typed_data/cli.rb
 - lib/typed_data/converter.rb
+- lib/typed_data/key_formatter.rb
+- lib/typed_data/restorer.rb
 - lib/typed_data/schema.rb
 - lib/typed_data/schema/array_type.rb
 - lib/typed_data/schema/boolean_type.rb
 - lib/typed_data/schema/bytes_type.rb
 - lib/typed_data/schema/enum_type.rb
+- lib/typed_data/schema/errors.rb
 - lib/typed_data/schema/float_type.rb
 - lib/typed_data/schema/int_type.rb
 - lib/typed_data/schema/long_type.rb
@@ -95,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A library that converts hash objects managed by an Avro schema