fluent-plugin-arrow 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9325c65f679d81a53d202a11a842a6c3a82b1cc3d600203064fc1329eeb3ded2
-  data.tar.gz: 134a1bb09054b3feeed480efb0d7df558ad8a8131dbe3951d73395511ed2e843
+  metadata.gz: 835c64e5e2e22f41142ec79b7ba525e7d47ca2fec9a70bec9daa579af86a6f2b
+  data.tar.gz: 8df48380e8a2f3eab3762ff0b91f88cba787cd0ac46cb1b9f02e69dfe99926eb
 SHA512:
-  metadata.gz: c72b02f95969d7dd8cdbfe525b579afc23a16f8fae1fdd72ac095479f94514c890287f7b3c41822888965a4f9554b33b5ed49e23d92401cefbfee3f8f8feb9b0
-  data.tar.gz: 18eaeb2135ecc0339e8986db79545a772aeeef0a2499119bfd3b2194beb125c5be518d6bb86dd0b5caaae1c1d842f98926557219c8fd6586fa04e0ac51b34e2c
+  metadata.gz: 1fff346a50875d7b554e067bf9b50dc64764059101b30cc9bdbadc515e45ab75841a35170f6339bea1b20d3e8365b8a7b617039757958e9bbf4f90b0c1e28348
+  data.tar.gz: bc034add73f3f31df0a6c1b6c24cea15eef0821f3be1062dbd940104cce178d105065b24b332eddc661faeaba9905bbc68f830c54647b10c68000f1bca4361a3

data/.dockerignore

@@ -0,0 +1,2 @@
+.bundle
+.git

data/.travis.yml

@@ -0,0 +1,10 @@
+sudo: required
+
+services:
+  - docker
+
+install:
+  - docker build -t joker1007/fluent-plugin-arrow .
+
+script:
+  - docker run joker1007/fluent-plugin-arrow /bin/sh -c "bundle exec rake"

data/Dockerfile

@@ -0,0 +1,14 @@
+FROM ruby:2.6-stretch
+
+RUN \
+  apt update && \
+  apt install -y apt-transport-https lsb-release && \
+  wget -O /usr/share/keyrings/apache-arrow-keyring.gpg \
+    https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-keyring.gpg && \
+  echo "deb [arch=amd64 signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main" > \
+    /etc/apt/sources.list.d/apache-arrow.list
+
+RUN mkdir /app
+WORKDIR /app
+COPY . /app
+RUN apt update && bundle install

data/README.md

@@ -49,10 +49,10 @@ You can generate configuration template:
       {"name": "key1", "type": "string"},
       {"name": "key2", "type": "uint64"},
       {"name": "key3", "type": "timestamp", "unit": "milli"},
-      {"name": "key4", "type": "list", "value_type": {"name": "value", "type": "uint64"}},
+      {"name": "key4", "type": "list", "field": {"name": "value", "type": "uint64"}},
       {"name": "key5", "type": "struct", "fields": [
         {"name": "bar1", "type": "uint64"},
-        {"name": "bar2", "type": "list", "value_type": {"name": "value", "type": "string"}}
+        {"name": "bar2", "type": "list", "field": {"name": "value", "type": "string"}}
       ]}
     ]
   </buffer>

data/fluent-plugin-arrow.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-arrow"
-  spec.version = "0.0.3"
+  spec.version = "0.0.4"
   spec.authors = ["joker1007"]
   spec.email   = ["kakyoin.hierophant@gmail.com"]
 
@@ -24,6 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
-  spec.add_runtime_dependency "red-arrow", ">= 0.10"
+  spec.add_runtime_dependency "red-arrow", ">= 0.12"
   spec.add_runtime_dependency "red-parquet"
 end

data/lib/fluent/plugin/buf_arrow_file.rb

@@ -17,7 +17,6 @@
 require "arrow"
 require 'fluent/plugin/buf_file'
 require 'fluent/plugin/buffer/arrow_file_chunk'
-require 'fluent/plugin/arrow/field_wrapper'
 
 module Fluent
   module Plugin
@@ -34,11 +33,7 @@ module Fluent
         super
 
         # [{"name" => foo1, "type" => "uint64"}, {"name" => foo2, "type" => "struct", "fields" => [{"name" => bar1, "type" => "string"}]}
-        @field_wrappers = @schema.each_with_object({}) do |field, h|
-          h[field["name"]] = Fluent::Plugin::Arrow::FieldWrapper.build(field)
-        end
-
-        @arrow_schema = ::Arrow::Schema.new(@field_wrappers.values.map(&:arrow_field))
+        @arrow_schema = ::Arrow::Schema.new(@schema)
       end
 
       def resume
@@ -48,9 +43,9 @@ module Fluent
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
         if @file_permission
-          chunk = Fluent::Plugin::Buffer::ArrowFileChunk.new(metadata, @path, :create, @arrow_schema, @field_wrappers, perm: @file_permission, chunk_size: @row_group_chunk_size, format: @arrow_format)
+          chunk = Fluent::Plugin::Buffer::ArrowFileChunk.new(metadata, @path, :create, @arrow_schema, perm: @file_permission, chunk_size: @row_group_chunk_size, format: @arrow_format)
         else
-          chunk = Fluent::Plugin::Buffer::ArrowFileChunk.new(metadata, @path, :create, @arrow_schema, @field_wrappers, chunk_size: @row_group_chunk_size, format: @arrow_format)
+          chunk = Fluent::Plugin::Buffer::ArrowFileChunk.new(metadata, @path, :create, @arrow_schema, chunk_size: @row_group_chunk_size, format: @arrow_format)
         end
 
         log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata

data/lib/fluent/plugin/buf_arrow_memory.rb

@@ -16,7 +16,6 @@
 require "arrow"
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buffer/arrow_memory_chunk'
-require 'fluent/plugin/arrow/field_wrapper'
 
 module Fluent
   module Plugin
@@ -33,11 +32,7 @@ module Fluent
         super
 
         # [{"name" => foo1, "type" => "uint64"}, {"name" => foo2, "type" => "struct", "fields" => [{"name" => bar1, "type" => "string"}]}
-        @field_wrappers = @schema.each_with_object({}) do |field, h|
-          h[field["name"]] = Fluent::Plugin::Arrow::FieldWrapper.build(field)
-        end
-
-        @arrow_schema = ::Arrow::Schema.new(@field_wrappers.values.map(&:arrow_field))
+        @arrow_schema = ::Arrow::Schema.new(@schema)
       end
 
       def resume
@@ -45,7 +40,7 @@ module Fluent
       end
 
       def generate_chunk(metadata)
-        Fluent::Plugin::Buffer::ArrowMemoryChunk.new(metadata, @arrow_schema, @field_wrappers, chunk_size: @row_group_chunk_size, format: @arrow_format)
+        Fluent::Plugin::Buffer::ArrowMemoryChunk.new(metadata, @arrow_schema, chunk_size: @row_group_chunk_size, format: @arrow_format)
       end
     end
   end

data/lib/fluent/plugin/buffer/arrow_buffer_string_builder.rb

@@ -10,32 +10,11 @@ module Fluent
         end
 
         def build_arrow_buffer_string
-          count = 0
-          each_record do |record|
-            count += 1
-            record.each do |k, v|
-              @field_wrappers[k].append(v)
-            end
-          end
+          record_batch = ::Arrow::RecordBatch.new(@schema, each_record)
           arrow_buf = ::Arrow::ResizableBuffer.new(bytesize * 1.2)
-
-          ::Arrow::BufferOutputStream.open(arrow_buf) do |output|
-            if @format == :parquet
-              Parquet::ArrowFileWriter.open(@schema, output) do |writer|
-                columns = @schema.fields.map do |f|
-                  ::Arrow::Column.new(f, @field_wrappers[f.name].finish)
-                end
-                table = ::Arrow::Table.new(@schema, columns)
-                writer.write_table(table, @chunk_size)
-              end
-            else
-              ::Arrow::RecordBatchFileWriter.open(output, @schema) do |writer|
-                record_batch = ::Arrow::RecordBatch.new(@schema, count, @field_wrappers.values.map(&:finish))
-                writer.write_record_batch(record_batch)
-              end
-            end
-          end
-
+          record_batch.to_table.save(arrow_buf,
+                                     format: @format,
+                                     chunk_size: @chunk_size)
           arrow_buf.data.to_s
         end
       end

data/lib/fluent/plugin/buffer/arrow_file_chunk.rb

@@ -19,7 +19,6 @@ require 'fluent/msgpack_factory'
 require 'fluent/plugin/buffer/chunk'
 require 'fluent/plugin/buffer/file_chunk'
 require 'fluent/plugin/buffer/arrow_buffer_string_builder'
-require 'fluent/plugin/arrow/field_wrapper'
 
 module Fluent
   module Plugin
@@ -27,10 +26,9 @@ module Fluent
       class ArrowFileChunk < FileChunk
         include ArrowBufferStringBuilder
 
-        def initialize(metadata, path, mode, schema, field_wrappers, perm: system_config.file_permission || FILE_PERMISSION, chunk_size: 1024, format: :arrow)
+        def initialize(metadata, path, mode, schema, perm: system_config.file_permission || FILE_PERMISSION, chunk_size: 1024, format: :arrow)
           super(metadata, path, mode, perm: perm, compress: :text)
           @schema = schema
-          @field_wrappers = field_wrappers
           @chunk_size = chunk_size
           @format = format
         end

data/lib/fluent/plugin/buffer/arrow_memory_chunk.rb

@@ -19,7 +19,6 @@ require 'fluent/msgpack_factory'
 require 'fluent/plugin/buffer/chunk'
 require 'fluent/plugin/buffer/memory_chunk'
 require 'fluent/plugin/buffer/arrow_buffer_string_builder'
-require 'fluent/plugin/arrow/field_wrapper'
 
 module Fluent
   module Plugin
@@ -27,10 +26,9 @@ module Fluent
       class ArrowMemoryChunk < MemoryChunk
         include ArrowBufferStringBuilder
 
-        def initialize(metadata, schema, field_wrappers, chunk_size: 1024, format: :arrow)
+        def initialize(metadata, schema, chunk_size: 1024, format: :arrow)
           super(metadata, compress: :text)
           @schema = schema
-          @field_wrappers = field_wrappers
           @chunk_size = chunk_size
           @format = format
         end

data/test/plugin/test_buf_arrow_memory.rb

@@ -25,8 +25,8 @@ class ArrowMemoryBufferTest < Test::Unit::TestCase
         {"name": "foo1", "type": "uint64"},
         {"name": "foo2", "type": "string"},
         {"name": "foo3", "type": "timestamp", "unit": "milli"},
-        {"name": "foo4", "type": "list", "value_type": {"name": "value", "type": "uint64"}},
-        {"name": "foo5", "type": "struct", "fields": [{"name": "bar1", "type": "uint64"}, {"name": "bar2", "type": "list", "value_type": {"name": "value", "type": "string"}}]}
+        {"name": "foo4", "type": "list", "field": {"name": "value", "type": "uint64"}},
+        {"name": "foo5", "type": "struct", "fields": [{"name": "bar1", "type": "uint64"}, {"name": "bar2", "type": "list", "field": {"name": "value", "type": "string"}}]}
       ]
     ]
     buffer_conf = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: :v1)

data/test/plugin/test_buffer_arrow_memory_chunk.rb

@@ -9,13 +9,8 @@ class ArrowMemoryChunkTest < Test::Unit::TestCase
       ::Arrow::Field.new("key2", :double),
       ::Arrow::Field.new("key3", ::Arrow::TimestampDataType.new(:second)),
     ]
-    field_wrappers = {
-      "key1" => Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "uint64"}),
-      "key2" => Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "double"}),
-      "key3" => Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "timestamp", "unit" => "second"}),
-    }
     @schema = Arrow::Schema.new(@fields)
-    @c = Fluent::Plugin::Buffer::ArrowMemoryChunk.new(Object.new, @schema, field_wrappers)
+    @c = Fluent::Plugin::Buffer::ArrowMemoryChunk.new(Object.new, @schema)
   end
 
   test "can #read" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-arrow
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-17 00:00:00.000000000 Z
+date: 2019-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -78,14 +78,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0.12'
 - !ruby/object:Gem::Dependency
   name: red-parquet
   requirement: !ruby/object:Gem::Requirement
@@ -107,13 +107,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".dockerignore"
 - ".gitignore"
+- ".travis.yml"
+- Dockerfile
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - fluent-plugin-arrow.gemspec
-- lib/fluent/plugin/arrow/field_wrapper.rb
 - lib/fluent/plugin/buf_arrow_file.rb
 - lib/fluent/plugin/buf_arrow_memory.rb
 - lib/fluent/plugin/buffer/arrow_buffer_string_builder.rb
@@ -123,7 +125,6 @@ files:
 - test/helper.rb
 - test/plugin/test_buf_arrow_memory.rb
 - test/plugin/test_buffer_arrow_memory_chunk.rb
-- test/plugin/test_field_wrapper.rb
 homepage: https://github.com/joker1007/fluent-plugin-arrow
 licenses:
 - Apache-2.0
@@ -143,8 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Apache Arrow formatter plugin for fluentd.
@@ -152,4 +152,3 @@ test_files:
 - test/helper.rb
 - test/plugin/test_buf_arrow_memory.rb
 - test/plugin/test_buffer_arrow_memory_chunk.rb
-- test/plugin/test_field_wrapper.rb

data/lib/fluent/plugin/arrow/field_wrapper.rb

@@ -1,259 +0,0 @@
-require "arrow"
-
-module Fluent
-  module Plugin
-    module Arrow
-      class FieldWrapper
-        class << self
-          def build(field)
-            case field["type"]
-            when "string"
-              StringFieldWrapper.new(field)
-            when "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64"
-              IntegerFieldWrapper.new(field)
-            when "float", "double"
-              FloatFieldWrapper.new(field)
-            when "boolean"
-              BooleanFieldWrapper.new(field)
-            when "date32"
-              Date32FieldWrapper.new(field)
-            when "date64"
-              Date64FieldWrapper.new(field)
-            when "timestamp"
-              TimestampFieldWrapper.new(field)
-            when "list"
-              ListFieldWrapper.new(field)
-            when "struct"
-              StructFieldWrapper.new(field)
-            else
-              raise "Unsupported data type"
-            end
-          end
-        end
-
-        attr_reader :field, :name, :type, :children, :arrow_field, :array_builder
-
-        def initialize(field)
-          @field = field
-          @name = field["name"]
-          @type = field["type"]
-          @children = []
-
-          field["value_type"]&.tap do |f|
-            @children << self.class.build(f)
-          end
-
-          field["fields"]&.each do |f|
-            @children << self.class.build(f)
-          end
-
-          create_arrow_field
-          create_array_builder
-        end
-
-        def append(value)
-          if value.nil?
-            @array_builder.append_null
-          else
-            @array_builder.append(cast_value(value))
-          end
-        end
-
-        def finish
-          @array_builder.finish
-        end
-
-        def create_arrow_field
-          @arrow_field = ::Arrow::Field.new(name, create_arrow_data_type)
-        end
-
-        def create_arrow_data_type
-          data_type_name = type.to_s.capitalize.gsub(/\AUint/, "UInt")
-          data_type_class_name = "#{data_type_name}DataType"
-          data_type_class = ::Arrow.const_get(data_type_class_name)
-          data_type_class.new
-        end
-
-        def create_array_builder(from_parent = nil)
-          if from_parent
-            @array_builder = from_parent
-          else
-            data_type_str = arrow_field.data_type.to_s
-            data_type_name = data_type_str.capitalize.gsub(/\AUint/, "UInt")
-            array_builder_class_name = "#{data_type_name}ArrayBuilder"
-            array_builder_class = ::Arrow.const_get(array_builder_class_name)
-            @array_builder = array_builder_class.new
-          end
-        end
-
-        def cast_value(value)
-          raise NotImplementedError
-        end
-      end
-
-      class StringFieldWrapper < FieldWrapper
-        def cast_value(value)
-          value.to_s
-        end
-      end
-
-      class IntegerFieldWrapper < FieldWrapper
-        def cast_value(value)
-          value.to_i
-        end
-      end
-
-      class FloatFieldWrapper < FieldWrapper
-        def cast_value(value)
-          value.to_f
-        end
-      end
-
-      class BooleanFieldWrapper < FieldWrapper
-        def cast_value(value)
-          !!value
-        end
-      end
-
-      require "date"
-      class Date32FieldWrapper < FieldWrapper
-        UNIX_EPOCH = Date.new(1970, 1, 1)
-        def cast_value(value)
-          date =
-            if value.respond_to?(:to_date)
-              value.to_date
-            else
-              Date.parse(value)
-            end
-
-          (date - UNIX_EPOCH).to_i
-        end
-
-        def create_array_builder(from_parent = nil)
-          if from_parent
-            @array_builder = from_parent
-          else
-            @array_builder = ::Arrow::Date32ArrayBuilder.new
-          end
-        end
-      end
-
-      class Date64FieldWrapper < FieldWrapper
-        UNIX_EPOCH = Date.new(1970, 1, 1)
-        def cast_value(value)
-          time =
-            if value.respond_to?(:to_time)
-              value.to_time
-            else
-              Time.parse(value)
-            end
-
-          time.to_i * 1_000 + time.usec / 1_000
-        end
-
-        def create_array_builder(from_parent = nil)
-          if from_parent
-            @array_builder = from_parent
-          else
-            @array_builder = ::Arrow::Date64ArrayBuilder.new
-          end
-        end
-      end
-
-      require "time"
-      class TimestampFieldWrapper < FieldWrapper
-        def cast_value(value)
-          value =
-            if value.is_a?(Fluent::EventTime)
-              Time.at(value, value.usec)
-            elsif value.respond_to?(:to_time)
-              value.to_time
-            elsif value.is_a?(String)
-              Time.parse(value)
-            else
-              value
-            end
-
-          return value if value.is_a?(Numeric)
-
-          case field["unit"]
-          when "second"
-            value.to_i
-          when "milli"
-            value.to_i * 1_000 + value.usec / 1_000
-          when "micro"
-            value.to_i * 1_000_000 + value.usec
-          else
-            value.to_i * 1_000_000_000 + value.nsec
-          end
-        end
-
-        def create_arrow_data_type
-          ::Arrow::TimestampDataType.new(field["unit"].to_sym)
-        end
-
-        def create_array_builder(from_parent = nil)
-          if from_parent
-            @array_builder = from_parent
-          else
-            @array_builder = ::Arrow::TimestampArrayBuilder.new(arrow_field.data_type)
-          end
-        end
-      end
-
-      class ListFieldWrapper < FieldWrapper
-        def append(value)
-          if value.nil?
-            @array_builder.append_null
-          else
-            @array_builder.append
-            value.each do |v|
-              @children[0].append(v)
-            end
-          end
-        end
-
-        def create_arrow_data_type
-          ::Arrow::ListDataType.new(children[0].arrow_field)
-        end
-
-        def create_array_builder(from_parent = nil)
-          if from_parent
-            @array_builder = from_parent
-          else
-            @array_builder = ::Arrow::ListArrayBuilder.new(arrow_field.data_type)
-          end
-
-          @children.each { |c| c.create_array_builder(@array_builder.value_builder) }
-        end
-      end
-
-      class StructFieldWrapper < FieldWrapper
-        def append(value)
-          if value.nil?
-            @array_builder.append_null
-          else
-            @array_builder.append
-            value.each do |k, v|
-              @children.find { |c| c.name == k }.append(v)
-            end
-          end
-        end
-
-        def create_arrow_data_type
-          ::Arrow::StructDataType.new(children.map(&:arrow_field))
-        end
-
-        def create_array_builder(from_parent = nil)
-          if from_parent
-            @array_builder = from_parent
-          else
-            @array_builder = ::Arrow::StructArrayBuilder.new(arrow_field.data_type)
-          end
-
-          @children.each_with_index { |c, i| c.create_array_builder(@array_builder.get_field_builder(i)) }
-        end
-      end
-    end
-  end
-end

data/test/plugin/test_field_wrapper.rb

@@ -1,145 +0,0 @@
-require "helper"
-require "fluent/plugin/arrow/field_wrapper"
-
-class ArrowFieldWrapperTest < Test::Unit::TestCase
-  test ".build (string)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "string"})
-    assert_equal "key1", field_wrapper.name
-    assert_equal "string", field_wrapper.type
-    assert_kind_of Arrow::Field, field_wrapper.arrow_field
-  end
-
-  test ".build (timestamp)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "timestamp", "unit" => "nano"})
-    assert_equal "key1", field_wrapper.name
-    assert_equal "timestamp", field_wrapper.type
-    assert_kind_of Arrow::Field, field_wrapper.arrow_field
-  end
-
-  test ".build (list)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "list", "value_type" => {"name" => "value", "type" => "string"}})
-    assert_equal "key1", field_wrapper.name
-    assert_equal "list", field_wrapper.type
-    assert_kind_of Arrow::Field, field_wrapper.arrow_field
-    assert_kind_of Arrow::ListDataType, field_wrapper.arrow_field.data_type
-    assert_kind_of Arrow::ListArrayBuilder, field_wrapper.array_builder
-
-    assert_equal "value", field_wrapper.children[0].name
-    assert_equal "string", field_wrapper.children[0].type
-    assert_kind_of Arrow::Field, field_wrapper.children[0].arrow_field
-    assert_kind_of Arrow::StringDataType, field_wrapper.children[0].arrow_field.data_type
-    assert_kind_of Arrow::StringArrayBuilder, field_wrapper.children[0].array_builder
-  end
-
-  test ".build (struct)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "struct", "fields" => [
-      {"name" => "foo1", "type" => "string"},
-      {"name" => "foo2", "type" => "uint64"},
-      {"name" => "foo3", "type" => "timestamp", "unit" => "milli"},
-    ]})
-    assert_equal "key1", field_wrapper.name
-    assert_equal "struct", field_wrapper.type
-    assert_kind_of Arrow::Field, field_wrapper.arrow_field
-    assert_kind_of Arrow::StructDataType, field_wrapper.arrow_field.data_type
-    assert_kind_of Arrow::StructArrayBuilder, field_wrapper.array_builder
-
-    assert_equal "foo1", field_wrapper.children[0].name
-    assert_equal "string", field_wrapper.children[0].type
-    assert_kind_of Arrow::Field, field_wrapper.children[0].arrow_field
-    assert_kind_of Arrow::StringDataType, field_wrapper.children[0].arrow_field.data_type
-    assert_kind_of Arrow::StringArrayBuilder, field_wrapper.children[0].array_builder
-
-    assert_equal "foo2", field_wrapper.children[1].name
-    assert_equal "uint64", field_wrapper.children[1].type
-    assert_kind_of Arrow::Field, field_wrapper.children[1].arrow_field
-    assert_kind_of Arrow::UInt64DataType, field_wrapper.children[1].arrow_field.data_type
-    assert_kind_of Arrow::UInt64ArrayBuilder, field_wrapper.children[1].array_builder
-
-    assert_equal "foo3", field_wrapper.children[2].name
-    assert_equal "timestamp", field_wrapper.children[2].type
-    assert_kind_of Arrow::Field, field_wrapper.children[2].arrow_field
-    assert_kind_of Arrow::TimestampDataType, field_wrapper.children[2].arrow_field.data_type
-    assert_kind_of Arrow::TimestampArrayBuilder, field_wrapper.children[2].array_builder
-  end
-
-  test ".build (nested)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "struct", "fields" => [
-      {"name" => "foo1", "type" => "string"},
-      {"name" => "foo2", "type" => "list", "value_type" => {"name" => "value", "type" => "uint64"}},
-    ]})
-    assert_equal "key1", field_wrapper.name
-    assert_equal "struct", field_wrapper.type
-    assert_kind_of Arrow::Field, field_wrapper.arrow_field
-    assert_kind_of Arrow::StructDataType, field_wrapper.arrow_field.data_type
-    assert_kind_of Arrow::StructArrayBuilder, field_wrapper.array_builder
-
-    assert_equal "foo1", field_wrapper.children[0].name
-    assert_equal "string", field_wrapper.children[0].type
-    assert_kind_of Arrow::Field, field_wrapper.children[0].arrow_field
-    assert_kind_of Arrow::StringDataType, field_wrapper.children[0].arrow_field.data_type
-    assert_kind_of Arrow::StringArrayBuilder, field_wrapper.children[0].array_builder
-
-    assert_equal "foo2", field_wrapper.children[1].name
-    assert_equal "list", field_wrapper.children[1].type
-    assert_kind_of Arrow::Field, field_wrapper.children[1].arrow_field
-    assert_kind_of Arrow::ListDataType, field_wrapper.children[1].arrow_field.data_type
-    assert_kind_of Arrow::ListArrayBuilder, field_wrapper.children[1].array_builder
-
-    assert_equal "value", field_wrapper.children[1].children[0].name
-    assert_equal "uint64", field_wrapper.children[1].children[0].type
-    assert_kind_of Arrow::Field, field_wrapper.children[1].children[0].arrow_field
-    assert_kind_of Arrow::UInt64DataType, field_wrapper.children[1].children[0].arrow_field.data_type
-    assert_kind_of Arrow::UInt64ArrayBuilder, field_wrapper.children[1].children[0].array_builder
-  end
-
-  test "#append (timestamp)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "timestamp", "unit" => "nano"})
-    time = Time.now
-    field_wrapper.append(time)
-    timestamp_array = field_wrapper.finish
-    assert_kind_of Time, timestamp_array[0]
-    assert_equal time.to_i, timestamp_array[0].to_i
-  end
-
-  test "#append (date32)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "date32"})
-    date = Date.today
-    field_wrapper.append(date)
-    date_array = field_wrapper.finish
-    assert_kind_of Date, date_array[0]
-    assert_equal date, date_array[0]
-  end
-
-  test "#append (date64)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "date64"})
-    date = Date.today
-    field_wrapper.append(date)
-    date_array = field_wrapper.finish
-    assert_kind_of DateTime, date_array[0]
-    assert_equal date, date_array[0].to_date
-  end
-
-  test "#append (nested)" do
-    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "struct", "fields" => [
-      {"name" => "foo1", "type" => "string"},
-      {"name" => "foo2", "type" => "list", "value_type" => {"name" => "value", "type" => "uint64"}},
-    ]})
-
-    field_wrapper.append({"foo1" => "rec1", "foo2" => [1, 2, 3]})
-    field_wrapper.append({"foo1" => "rec2", "foo2" => [4, 5]})
-
-    struct_array = field_wrapper.finish
-    assert_kind_of Arrow::StringArray, struct_array.fields[0]
-    assert_equal "rec1", struct_array.fields[0][0]
-    assert_equal "rec2", struct_array.fields[0][1]
-
-    assert_kind_of Arrow::UInt64Array, struct_array.fields[1].get_value(0)
-    assert_equal 1, struct_array.fields[1].get_value(0)[0]
-    assert_equal 2, struct_array.fields[1].get_value(0)[1]
-    assert_equal 3, struct_array.fields[1].get_value(0)[2]
-
-    assert_kind_of Arrow::UInt64Array, struct_array.fields[1].get_value(1)
-    assert_equal 4, struct_array.fields[1].get_value(1)[0]
-    assert_equal 5, struct_array.fields[1].get_value(1)[1]
-  end
-end