fluent-plugin-arrow 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c46290f5f4e9c75defd5494117f6deb504dad5fb70c51ceaccb5aabe9ff06934
-  data.tar.gz: 6bc16d3413997b32efe5233a3aea4e3f4a3020eaf70c10675d8d1eef5899f0a2
+  metadata.gz: a1ed6b1b40e55501097a3a0455b2f2be93b3cf55d7e8cb831d4dc742f4007b97
+  data.tar.gz: fdcaed4462a919076c29f5947c3c81744b887e0099d2e5ad7a317e6845391da1
 SHA512:
-  metadata.gz: 73e46c999a9a62710c1ead926565d18391cd377e61e6eb01bb76d6ca102cf8423ffd01c7934dfd6fb0c32282bf04867810854dd14b76ec79416597e6237aae76
-  data.tar.gz: 6c14dc63fa79ece11140fd1b2ee916bcf2917260b0ae9a5741024ea5923d18e3a66add447a7f8c3cf1d5d24fccb4c8cb651d40e609f353813f7673d39669cf84
+  metadata.gz: 5f219215b1f8b8a57b2f9d79a6d1fdec39e10652c6d674a8d1390f7f3ce75d8752c15e89b268e36fb781bed44cfed2e128fd840b0978eb508c75dec1fc2b82a1
+  data.tar.gz: 2d473fbead6b6c12e361b045e7de325652b36c22d94685be289bc8a076c03f4dcb8b2c5270d922f76c0f462475d6846c96176abf4085abf24affaad0bc4b04fd

data/README.md

@@ -1,8 +1,13 @@
 # fluent-plugin-arrow
 
-[Fluentd](https://fluentd.org/) formatter plugin to do something.
+[Fluentd](https://fluentd.org/) buffer plugin to output Apache Arrow and Parquet format.
 
-TODO: write description for you plugin.
+## Prerequisite
+
+- [Apache Arrow c++](https://github.com/apache/arrow/tree/master/cpp) (with -DARROW_PARQUET=ON)
+- [Apache Arrow c_glib](https://github.com/apache/arrow/tree/master/c_glib)
+- [red-arrow](https://github.com/apache/arrow/tree/master/ruby/red-arrow)
+- [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet)
 
 ## Installation
 
@@ -31,7 +36,31 @@ $ bundle
 You can generate configuration template:
 
 ```
-$ fluent-plugin-config-format formatter arrow
+<match arrow>
+  @type file
+
+  path arrow_test
+
+  <buffer>
+    @type arrow_memory
+    arrow_format arrow # or parquet
+
+    schema [
+      {"name": "key1", "type": "string"},
+      {"name": "key2", "type": "uint64"},
+      {"name": "key3", "type": "timestamp", "unit": "milli"},
+      {"name": "key4", "type": "list", "value_type": {"name": "value", "type": "uint64"}},
+      {"name": "key5", "type": "struct", "fields": [
+        {"name": "bar1", "type": "uint64"},
+        {"name": "bar2", "type": "list", "value_type": {"name": "value", "type": "string"}}
+      ]}
+    ]
+  </buffer>
+
+  <format>
+    @type arrow
+  </format>
+</match>
 ```
 
 You can copy and paste generated documents here.

data/fluent-plugin-arrow.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-arrow"
-  spec.version = "0.0.1"
+  spec.version = "0.0.2"
   spec.authors = ["joker1007"]
   spec.email   = ["kakyoin.hierophant@gmail.com"]
 

data/lib/fluent/plugin/arrow/field_wrapper.rb

@@ -0,0 +1,259 @@
+require "arrow"
+
+module Fluent
+  module Plugin
+    module Arrow
+      class FieldWrapper
+        class << self
+          def build(field)
+            case field["type"]
+            when "string"
+              StringFieldWrapper.new(field)
+            when "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64"
+              IntegerFieldWrapper.new(field)
+            when "float", "double"
+              FloatFieldWrapper.new(field)
+            when "boolean"
+              BooleanFieldWrapper.new(field)
+            when "date32"
+              Date32FieldWrapper.new(field)
+            when "date64"
+              Date64FieldWrapper.new(field)
+            when "timestamp"
+              TimestampFieldWrapper.new(field)
+            when "list"
+              ListFieldWrapper.new(field)
+            when "struct"
+              StructFieldWrapper.new(field)
+            else
+              raise "Unsupported data type"
+            end
+          end
+        end
+
+        attr_reader :field, :name, :type, :children, :arrow_field, :array_builder
+
+        def initialize(field)
+          @field = field
+          @name = field["name"]
+          @type = field["type"]
+          @children = []
+
+          field["value_type"]&.tap do |f|
+            @children << self.class.build(f)
+          end
+
+          field["fields"]&.each do |f|
+            @children << self.class.build(f)
+          end
+
+          create_arrow_field
+          create_array_builder
+        end
+
+        def append(value)
+          if value.nil?
+            @array_builder.append_null
+          else
+            @array_builder.append(cast_value(value))
+          end
+        end
+
+        def finish
+          @array_builder.finish
+        end
+
+        def create_arrow_field
+          @arrow_field = ::Arrow::Field.new(name, create_arrow_data_type)
+        end
+
+        def create_arrow_data_type
+          data_type_name = type.to_s.capitalize.gsub(/\AUint/, "UInt")
+          data_type_class_name = "#{data_type_name}DataType"
+          data_type_class = ::Arrow.const_get(data_type_class_name)
+          data_type_class.new
+        end
+
+        def create_array_builder(from_parent = nil)
+          if from_parent
+            @array_builder = from_parent
+          else
+            data_type_str = arrow_field.data_type.to_s
+            data_type_name = data_type_str.capitalize.gsub(/\AUint/, "UInt")
+            array_builder_class_name = "#{data_type_name}ArrayBuilder"
+            array_builder_class = ::Arrow.const_get(array_builder_class_name)
+            @array_builder = array_builder_class.new
+          end
+        end
+
+        def cast_value(value)
+          raise NotImplementedError
+        end
+      end
+
+      class StringFieldWrapper < FieldWrapper
+        def cast_value(value)
+          value.to_s
+        end
+      end
+
+      class IntegerFieldWrapper < FieldWrapper
+        def cast_value(value)
+          value.to_i
+        end
+      end
+
+      class FloatFieldWrapper < FieldWrapper
+        def cast_value(value)
+          value.to_f
+        end
+      end
+
+      class BooleanFieldWrapper < FieldWrapper
+        def cast_value(value)
+          !!value
+        end
+      end
+
+      require "date"
+      class Date32FieldWrapper < FieldWrapper
+        UNIX_EPOCH = Date.new(1970, 1, 1)
+        def cast_value(value)
+          date =
+            if value.respond_to?(:to_date)
+              value.to_date
+            else
+              Date.parse(value)
+            end
+
+          (date - UNIX_EPOCH).to_i
+        end
+
+        def create_array_builder(from_parent = nil)
+          if from_parent
+            @array_builder = from_parent
+          else
+            @array_builder = ::Arrow::Date32ArrayBuilder.new
+          end
+        end
+      end
+
+      class Date64FieldWrapper < FieldWrapper
+        UNIX_EPOCH = Date.new(1970, 1, 1)
+        def cast_value(value)
+          time =
+            if value.respond_to?(:to_time)
+              value.to_time
+            else
+              Time.parse(value)
+            end
+
+          time.to_i * 1_000 + time.usec / 1_000
+        end
+
+        def create_array_builder(from_parent = nil)
+          if from_parent
+            @array_builder = from_parent
+          else
+            @array_builder = ::Arrow::Date64ArrayBuilder.new
+          end
+        end
+      end
+
+      require "time"
+      class TimestampFieldWrapper < FieldWrapper
+        def cast_value(value)
+          value =
+            if value.is_a?(Fluent::EventTime)
+              Time.at(value, value.usec)
+            elsif value.respond_to?(:to_time)
+              value.to_time
+            elsif value.is_a?(String)
+              Time.parse(value)
+            else
+              value
+            end
+
+          return value if value.is_a?(Numeric)
+
+          case field["unit"]
+          when "second"
+            value.to_i
+          when "milli"
+            value.to_i * 1_000 + value.usec / 1_000
+          when "micro"
+            value.to_i * 1_000_000 + value.usec
+          else
+            value.to_i * 1_000_000_000 + value.nsec
+          end
+        end
+
+        def create_arrow_data_type
+          ::Arrow::TimestampDataType.new(field["unit"].to_sym)
+        end
+
+        def create_array_builder(from_parent = nil)
+          if from_parent
+            @array_builder = from_parent
+          else
+            @array_builder = ::Arrow::TimestampArrayBuilder.new(arrow_field.data_type)
+          end
+        end
+      end
+
+      class ListFieldWrapper < FieldWrapper
+        def append(value)
+          if value.nil?
+            @array_builder.append_null
+          else
+            @array_builder.append
+            value.each do |v|
+              @children[0].append(v)
+            end
+          end
+        end
+
+        def create_arrow_data_type
+          ::Arrow::ListDataType.new(children[0].arrow_field)
+        end
+
+        def create_array_builder(from_parent = nil)
+          if from_parent
+            @array_builder = from_parent
+          else
+            @array_builder = ::Arrow::ListArrayBuilder.new(arrow_field.data_type)
+          end
+
+          @children.each { |c| c.create_array_builder(@array_builder.value_builder) }
+        end
+      end
+
+      class StructFieldWrapper < FieldWrapper
+        def append(value)
+          if value.nil?
+            @array_builder.append_null
+          else
+            @array_builder.append
+            value.each do |k, v|
+              @children.find { |c| c.name == k }.append(v)
+            end
+          end
+        end
+
+        def create_arrow_data_type
+          ::Arrow::StructDataType.new(children.map(&:arrow_field))
+        end
+
+        def create_array_builder(from_parent = nil)
+          if from_parent
+            @array_builder = from_parent
+          else
+            @array_builder = ::Arrow::StructArrayBuilder.new(arrow_field.data_type)
+          end
+
+          @children.each_with_index { |c, i| c.create_array_builder(@array_builder.get_field_builder(i)) }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/buf_arrow_memory.rb

@@ -16,6 +16,7 @@
 require "arrow"
 require 'fluent/plugin/buffer'
 require 'fluent/plugin/buffer/arrow_memory_chunk'
+require 'fluent/plugin/arrow/field_wrapper'
 
 module Fluent
   module Plugin
@@ -32,11 +33,11 @@ module Fluent
         super
 
         # [{"name" => foo1, "type" => "uint64"}, {"name" => foo2, "type" => "struct", "fields" => [{"name" => bar1, "type" => "string"}]}
-        arrow_fields = @schema.map do |field|
-          create_arrow_field(field)
+        @field_wrappers = @schema.each_with_object({}) do |field, h|
+          h[field["name"]] = Fluent::Plugin::Arrow::FieldWrapper.build(field)
         end
 
-        @arrow_schema = Arrow::Schema.new(arrow_fields)
+        @arrow_schema = ::Arrow::Schema.new(@field_wrappers.values.map(&:arrow_field))
       end
 
       def resume
@@ -44,29 +45,7 @@ module Fluent
       end
 
       def generate_chunk(metadata)
-        Fluent::Plugin::Buffer::ArrowMemoryChunk.new(metadata, @arrow_schema, chunk_size: @row_group_chunk_size, format: @arrow_format)
-      end
-
-      private
-
-      def create_arrow_field(field)
-        Arrow::Field.new(field["name"], create_arrow_data_type(field))
-      end
-
-      def create_arrow_data_type(field)
-        case field["type"]
-        when "struct"
-          Arrow::StructDataType.new(field["fields"].map { |f| create_arrow_field(f) })
-        when "list"
-          Arrow::ListDataType.new(create_arrow_field(field["value_type"]))
-        when "timestamp"
-          Arrow::TimestampDataType.new(field["unit"].to_sym)
-        else
-          data_type_name = field["type"].to_s.capitalize.gsub(/\AUint/, "UInt")
-          data_type_class_name = "#{data_type_name}DataType"
-          data_type_class = Arrow.const_get(data_type_class_name)
-          data_type_class.new
-        end
+        Fluent::Plugin::Buffer::ArrowMemoryChunk.new(metadata, @arrow_schema, @field_wrappers, chunk_size: @row_group_chunk_size, format: @arrow_format)
       end
     end
   end

data/lib/fluent/plugin/buffer/arrow_memory_chunk.rb

@@ -15,22 +15,21 @@
 
 require 'arrow'
 require 'parquet'
+require 'fluent/msgpack_factory'
 require 'fluent/plugin/buffer/chunk'
 require 'fluent/plugin/buffer/memory_chunk'
+require 'fluent/plugin/arrow/field_wrapper'
 
 module Fluent
   module Plugin
     class Buffer
       class ArrowMemoryChunk < MemoryChunk
-        def initialize(metadata, schema, chunk_size: 1024, format: :arrow)
+        def initialize(metadata, schema, field_wrappers, chunk_size: 1024, format: :arrow)
           super(metadata, compress: :text)
           @schema = schema
+          @field_wrappers = field_wrappers
           @chunk_size = chunk_size
           @format = format
-          @array_builders = {}
-          @schema.fields.each do |f|
-            @array_builders[f.name] = field_to_array_builder(f)
-          end
           @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker
         end
 
@@ -49,48 +48,28 @@ module Fluent
 
         private
 
-        def field_to_array_builder(f)
-          data_type_str = f.data_type.to_s
-          if data_type_str =~ /timestamp/
-            return Arrow::TimestampArrayBuilder.new(f.data_type)
-          end
-
-          data_type_name = data_type_str.capitalize.gsub(/\AUint/, "UInt")
-          array_builder_class_name = "#{data_type_name}ArrayBuilder"
-          array_builder_class = Arrow.const_get(array_builder_class_name)
-          if array_builder_class.method(:new).arity > 0
-            array_builder_class.new(f.data_type)
-          else
-            array_builder_class.new
-          end
-        end
-
         def build_arrow_buffer_string
           count = 0
           @unpacker.feed_each(@chunk) do |record|
             count += 1
             record.each do |k, v|
-              if v.nil?
-                @array_builders[k].append_null
-              else
-                @array_builders[k].append(v)
-              end
+              @field_wrappers[k].append(v)
             end
           end
-          arrow_buf = Arrow::ResizableBuffer.new(@chunk_bytes * 1.2)
+          arrow_buf = ::Arrow::ResizableBuffer.new(@chunk_bytes * 1.2)
 
-          Arrow::BufferOutputStream.open(arrow_buf) do |output|
+          ::Arrow::BufferOutputStream.open(arrow_buf) do |output|
             if @format == :parquet
               Parquet::ArrowFileWriter.open(@schema, output) do |writer|
                 columns = @schema.fields.map do |f|
-                  Arrow::Column.new(f, @array_builders[f.name].finish)
+                  ::Arrow::Column.new(f, @field_wrappers[f.name].finish)
                 end
-                table = Arrow::Table.new(@schema, columns)
+                table = ::Arrow::Table.new(@schema, columns)
                 writer.write_table(table, @chunk_size)
               end
             else
-              Arrow::RecordBatchFileWriter.open(output, @schema) do |writer|
-                record_batch = Arrow::RecordBatch.new(@schema, count, @array_builders.values.map(&:finish))
+              ::Arrow::RecordBatchFileWriter.open(output, @schema) do |writer|
+                record_batch = ::Arrow::RecordBatch.new(@schema, count, @field_wrappers.values.map(&:finish))
                 writer.write_record_batch(record_batch)
               end
             end

data/test/plugin/test_buffer_arrow_memory_chunk.rb

@@ -5,12 +5,17 @@ require "fluent/plugin/buffer/arrow_memory_chunk"
 class ArrowMemoryChunkTest < Test::Unit::TestCase
   setup do
     @fields = [
-      Arrow::Field.new("key1", :uint64),
-      Arrow::Field.new("key2", :double),
-      Arrow::Field.new("key3", Arrow::TimestampDataType.new(:second)),
+      ::Arrow::Field.new("key1", :uint64),
+      ::Arrow::Field.new("key2", :double),
+      ::Arrow::Field.new("key3", ::Arrow::TimestampDataType.new(:second)),
     ]
+    field_wrappers = {
+      "key1" => Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "uint64"}),
+      "key2" => Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "double"}),
+      "key3" => Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "timestamp", "unit" => "second"}),
+    }
     @schema = Arrow::Schema.new(@fields)
-    @c = Fluent::Plugin::Buffer::ArrowMemoryChunk.new(Object.new, @schema)
+    @c = Fluent::Plugin::Buffer::ArrowMemoryChunk.new(Object.new, @schema, field_wrappers)
   end
 
   test "can #read" do
@@ -18,44 +23,41 @@ class ArrowMemoryChunkTest < Test::Unit::TestCase
     d2 = {"key1" => 124, "key2" => 11.1234, "key3" => Fluent::EventTime.from_time(Time.now)}
     data = [d1.to_msgpack, d2.to_msgpack]
     @c.append(data)
-    Arrow::BufferInputStream.open(Arrow::Buffer.new(@c.read)) do |input|
-      reader = Arrow::RecordBatchFileReader.new(input)
+    ::Arrow::BufferInputStream.open(::Arrow::Buffer.new(@c.read)) do |input|
+      reader = ::Arrow::RecordBatchFileReader.new(input)
 
       reader.each do |record_batch|
         assert { record_batch.n_rows == 2 }
 
-        assert { record_batch.find_column(@fields[0].name).class == Arrow::UInt64Array }
+        assert { record_batch.find_column(@fields[0].name).class == ::Arrow::UInt64Array }
         assert { record_batch.find_column(@fields[0].name).values == [123, 124] }
       end
     end
   end
 
   test "can #write_to" do
-    d1 = {"key1" => 123, "key2" => 10.1234, "key3" => Fluent::EventTime.from_time(Time.now)}
-    d2 = {"key1" => 124, "key2" => 11.1234, "key3" => Fluent::EventTime.from_time(Time.now)}
+    time = Time.now
+    d1 = {"key1" => 123, "key2" => 10.1234, "key3" => Fluent::EventTime.from_time(time)}
+    d2 = {"key1" => 124, "key2" => 11.1234, "key3" => Fluent::EventTime.from_time(time)}
     data = [d1.to_msgpack, d2.to_msgpack]
     @c.append(data)
     Tempfile.create do |tf|
       @c.write_to(tf)
       tf.flush
 
-      Arrow::MemoryMappedInputStream.open(tf.path) do |input|
-        reader = Arrow::RecordBatchFileReader.new(input)
+      ::Arrow::MemoryMappedInputStream.open(tf.path) do |input|
+        reader = ::Arrow::RecordBatchFileReader.new(input)
         reader.each_with_index do |record_batch, i|
           reader.each do |record_batch|
             assert { record_batch.n_rows == 2 }
 
-            assert { record_batch.find_column(@fields[0].name).class == Arrow::UInt64Array }
+            assert { record_batch.find_column(@fields[0].name).class == ::Arrow::UInt64Array }
             assert { record_batch.find_column(@fields[0].name).values == [123, 124] }
+            assert { record_batch.find_column(@fields[1].name).values == [10.1234, 11.1234] }
+            assert { record_batch.find_column(@fields[2].name)[0].to_i == time.to_i }
           end
         end
       end
     end
   end
-
-  private
-
-  def create_driver(conf)
-    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::ArrowFormatter).configure(conf)
-  end
 end

data/test/plugin/test_field_wrapper.rb

@@ -0,0 +1,145 @@
+require "helper"
+require "fluent/plugin/arrow/field_wrapper"
+
+class ArrowFieldWrapperTest < Test::Unit::TestCase
+  test ".build (string)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "string"})
+    assert_equal "key1", field_wrapper.name
+    assert_equal "string", field_wrapper.type
+    assert_kind_of Arrow::Field, field_wrapper.arrow_field
+  end
+
+  test ".build (timestamp)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "timestamp", "unit" => "nano"})
+    assert_equal "key1", field_wrapper.name
+    assert_equal "timestamp", field_wrapper.type
+    assert_kind_of Arrow::Field, field_wrapper.arrow_field
+  end
+
+  test ".build (list)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "list", "value_type" => {"name" => "value", "type" => "string"}})
+    assert_equal "key1", field_wrapper.name
+    assert_equal "list", field_wrapper.type
+    assert_kind_of Arrow::Field, field_wrapper.arrow_field
+    assert_kind_of Arrow::ListDataType, field_wrapper.arrow_field.data_type
+    assert_kind_of Arrow::ListArrayBuilder, field_wrapper.array_builder
+
+    assert_equal "value", field_wrapper.children[0].name
+    assert_equal "string", field_wrapper.children[0].type
+    assert_kind_of Arrow::Field, field_wrapper.children[0].arrow_field
+    assert_kind_of Arrow::StringDataType, field_wrapper.children[0].arrow_field.data_type
+    assert_kind_of Arrow::StringArrayBuilder, field_wrapper.children[0].array_builder
+  end
+
+  test ".build (struct)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "struct", "fields" => [
+      {"name" => "foo1", "type" => "string"},
+      {"name" => "foo2", "type" => "uint64"},
+      {"name" => "foo3", "type" => "timestamp", "unit" => "milli"},
+    ]})
+    assert_equal "key1", field_wrapper.name
+    assert_equal "struct", field_wrapper.type
+    assert_kind_of Arrow::Field, field_wrapper.arrow_field
+    assert_kind_of Arrow::StructDataType, field_wrapper.arrow_field.data_type
+    assert_kind_of Arrow::StructArrayBuilder, field_wrapper.array_builder
+
+    assert_equal "foo1", field_wrapper.children[0].name
+    assert_equal "string", field_wrapper.children[0].type
+    assert_kind_of Arrow::Field, field_wrapper.children[0].arrow_field
+    assert_kind_of Arrow::StringDataType, field_wrapper.children[0].arrow_field.data_type
+    assert_kind_of Arrow::StringArrayBuilder, field_wrapper.children[0].array_builder
+
+    assert_equal "foo2", field_wrapper.children[1].name
+    assert_equal "uint64", field_wrapper.children[1].type
+    assert_kind_of Arrow::Field, field_wrapper.children[1].arrow_field
+    assert_kind_of Arrow::UInt64DataType, field_wrapper.children[1].arrow_field.data_type
+    assert_kind_of Arrow::UInt64ArrayBuilder, field_wrapper.children[1].array_builder
+
+    assert_equal "foo3", field_wrapper.children[2].name
+    assert_equal "timestamp", field_wrapper.children[2].type
+    assert_kind_of Arrow::Field, field_wrapper.children[2].arrow_field
+    assert_kind_of Arrow::TimestampDataType, field_wrapper.children[2].arrow_field.data_type
+    assert_kind_of Arrow::TimestampArrayBuilder, field_wrapper.children[2].array_builder
+  end
+
+  test ".build (nested)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "struct", "fields" => [
+      {"name" => "foo1", "type" => "string"},
+      {"name" => "foo2", "type" => "list", "value_type" => {"name" => "value", "type" => "uint64"}},
+    ]})
+    assert_equal "key1", field_wrapper.name
+    assert_equal "struct", field_wrapper.type
+    assert_kind_of Arrow::Field, field_wrapper.arrow_field
+    assert_kind_of Arrow::StructDataType, field_wrapper.arrow_field.data_type
+    assert_kind_of Arrow::StructArrayBuilder, field_wrapper.array_builder
+
+    assert_equal "foo1", field_wrapper.children[0].name
+    assert_equal "string", field_wrapper.children[0].type
+    assert_kind_of Arrow::Field, field_wrapper.children[0].arrow_field
+    assert_kind_of Arrow::StringDataType, field_wrapper.children[0].arrow_field.data_type
+    assert_kind_of Arrow::StringArrayBuilder, field_wrapper.children[0].array_builder
+
+    assert_equal "foo2", field_wrapper.children[1].name
+    assert_equal "list", field_wrapper.children[1].type
+    assert_kind_of Arrow::Field, field_wrapper.children[1].arrow_field
+    assert_kind_of Arrow::ListDataType, field_wrapper.children[1].arrow_field.data_type
+    assert_kind_of Arrow::ListArrayBuilder, field_wrapper.children[1].array_builder
+
+    assert_equal "value", field_wrapper.children[1].children[0].name
+    assert_equal "uint64", field_wrapper.children[1].children[0].type
+    assert_kind_of Arrow::Field, field_wrapper.children[1].children[0].arrow_field
+    assert_kind_of Arrow::UInt64DataType, field_wrapper.children[1].children[0].arrow_field.data_type
+    assert_kind_of Arrow::UInt64ArrayBuilder, field_wrapper.children[1].children[0].array_builder
+  end
+
+  test "#append (timestamp)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "timestamp", "unit" => "nano"})
+    time = Time.now
+    field_wrapper.append(time)
+    timestamp_array = field_wrapper.finish
+    assert_kind_of Time, timestamp_array[0]
+    assert_equal time.to_i, timestamp_array[0].to_i
+  end
+
+  test "#append (date32)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "date32"})
+    date = Date.today
+    field_wrapper.append(date)
+    date_array = field_wrapper.finish
+    assert_kind_of Date, date_array[0]
+    assert_equal date, date_array[0]
+  end
+
+  test "#append (date64)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "date64"})
+    date = Date.today
+    field_wrapper.append(date)
+    date_array = field_wrapper.finish
+    assert_kind_of DateTime, date_array[0]
+    assert_equal date, date_array[0].to_date
+  end
+
+  test "#append (nested)" do
+    field_wrapper = Fluent::Plugin::Arrow::FieldWrapper.build({"name" => "key1", "type" => "struct", "fields" => [
+      {"name" => "foo1", "type" => "string"},
+      {"name" => "foo2", "type" => "list", "value_type" => {"name" => "value", "type" => "uint64"}},
+    ]})
+
+    field_wrapper.append({"foo1" => "rec1", "foo2" => [1, 2, 3]})
+    field_wrapper.append({"foo1" => "rec2", "foo2" => [4, 5]})
+
+    struct_array = field_wrapper.finish
+    assert_kind_of Arrow::StringArray, struct_array.fields[0]
+    assert_equal "rec1", struct_array.fields[0][0]
+    assert_equal "rec2", struct_array.fields[0][1]
+
+    assert_kind_of Arrow::UInt64Array, struct_array.fields[1].get_value(0)
+    assert_equal 1, struct_array.fields[1].get_value(0)[0]
+    assert_equal 2, struct_array.fields[1].get_value(0)[1]
+    assert_equal 3, struct_array.fields[1].get_value(0)[2]
+
+    assert_kind_of Arrow::UInt64Array, struct_array.fields[1].get_value(1)
+    assert_equal 4, struct_array.fields[1].get_value(1)[0]
+    assert_equal 5, struct_array.fields[1].get_value(1)[1]
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-arrow
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-28 00:00:00.000000000 Z
+date: 2018-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -113,13 +113,14 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-arrow.gemspec
+- lib/fluent/plugin/arrow/field_wrapper.rb
 - lib/fluent/plugin/buf_arrow_memory.rb
 - lib/fluent/plugin/buffer/arrow_memory_chunk.rb
 - lib/fluent/plugin/formatter_arrow.rb
 - test/helper.rb
 - test/plugin/test_buf_arrow_memory.rb
 - test/plugin/test_buffer_arrow_memory_chunk.rb
-- test/plugin/test_formatter_arrow.rb
+- test/plugin/test_field_wrapper.rb
 homepage: https://github.com/joker1007/fluent-plugin-arrow
 licenses:
 - Apache-2.0
@@ -148,4 +149,4 @@ test_files:
 - test/helper.rb
 - test/plugin/test_buf_arrow_memory.rb
 - test/plugin/test_buffer_arrow_memory_chunk.rb
-- test/plugin/test_formatter_arrow.rb
+- test/plugin/test_field_wrapper.rb

data/test/plugin/test_formatter_arrow.rb

@@ -1,14 +0,0 @@
-require "helper"
-require "fluent/plugin/formatter_arrow.rb"
-
-class ArrowFormatterTest < Test::Unit::TestCase
-  setup do
-    Fluent::Test.setup
-  end
-
-  private
-
-  def create_driver(conf)
-    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::ArrowFormatter).configure(conf)
-  end
-end