fluent-plugin-parser-avro 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: accbc87b882c2e1e0999a9992da794fbde6be745f240e043e77e92d9964406ac
-  data.tar.gz: 817d393998e57f8ba2352f405e565b0c59de34e3c92b7cbc38653453a621fd4a
+  metadata.gz: 9633a316c1de1e4cb83b487d99d5a3814f1cde2b96d20633cfe728c584108761
+  data.tar.gz: 6438fe5e248c98540602183a5828a7dbc596fe5000c4993fd428ee9223ed3077
 SHA512:
-  metadata.gz: 2dda0e434a73d6e3d1380c3e96e5cfe1595c3041bfe4afa9b99b125bcc8b19c7177139860ea108c7523f418d5fd589bc5edcd673d3025f628c0b47f2ca664c31
-  data.tar.gz: 4277e90990b38c9d72e5982f817d02c39f57592af64f2b03c54af9ec708b57908f3a13bd41de3ad2a5e82794b4612b555888ace4e10ec197d728e84b89d4718e
+  metadata.gz: 8b8894c17aaa33916ef3601634d36139f2817db875eba2c5e552a0309f2c5f3ce6578526c5aad54244cd51ce36cf30bc04f64ce576fcbd4ccb89fc7726d4102d
+  data.tar.gz: a9dbc9e144b1e0f7f2b585a51c0519e6c2953d7258b5bd13fd8d6246754f9956bffcee43777378692ea4ef2d8997d8f6648653d8928e2345e70c32e48ab849a6

data/LICENSE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2020- Hiroshi Hatake
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/README.md

@@ -32,12 +32,22 @@ $ bundle
 * **schema_file** (string) (optional): avro schema file path.
 * **schema_json** (string) (optional): avro schema definition hash.
 * **schema_url** (string) (optional): avro schema remote URL.
-* **schema_registery_with_subject_url** (string) (optional): avro schema registry URL.
 * **schema_url_key** (string) (optional): avro schema registry or something's response schema key.
 * **writers_schema_file** (string) (optional): avro schema file path for writers definition.
 * **writers_schema_json** (string) (optional): avro schema definition hash for writers definition.
 * **readers_schema_file** (string) (optional): avro schema file path for readers definition.
 * **readers_schema_json** (string) (optional): avro schema definition hash for readers definition.
+* **use_confluent_schema** (bool) (optional): Assume to use confluent schema. Confluent avro schema uses the first 5-bytes for magic byte (1 byte) and schema_id (4 bytes). This parameter specifies to skip reading the first 5-bytes or not.
+  * Default value: `true`.
+
+### \<confluent_registry\> section (optional) (single)
+
+* **url** (string) (required): confluent schema registry URL.
+* **subject** (string) (required): Specify schema subject.
+* **schema_key** (string) (optional): Specify schema key on confluent registry REST API response.
+  * Default value: `schema`.
+* **schema_version** (string) (optional): Specify schema version for the specified subject.
+  * Default value: `latest`.
 
 ### Configuration Example
 
@@ -48,7 +58,14 @@ $ bundle
   # schema_json { "namespace": "org.fluentd.parser.avro", "type": "record", "name": "User", "fields" : [{"name": "username", "type": "string"}, {"name": "age", "type": "int"}, {"name": "verified", "type": ["boolean", "null"], "default": false}]}
   # schema_url http(s)://[server fqdn]:[port]/subjects/[a great user's subject]/[the latest schema version]
   # schema_key schema
-  # schema_registery_with_subject_url http(s)://[server fqdn]:[port]/subjects/[a great user's subject]/
+  # When using with confluent registry without <confluent_registry>, this parameter must be true.
+  # use_confluent_schema true
+  #<confluent_registry>
+  #  url http://localhost:8081/
+  #  subject your-awesome-subject
+  #  # schema_key schema
+  #  # schema_version 1
+  #</confluent_registry>
 </parse>
 ```
 
@@ -58,22 +75,44 @@ Confluent AVRO schema registry should respond with REST API.
 
 This plugin uses the following API:
 
-* [`GET /subjects/(string: subject)/versions`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--subjects-(string-%20subject)-versions)
 * [`GET /subjects/(string: subject)/versions/(versionId: version)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--subjects-(string-%20subject)-versions)
 
-Users can specify a URL for retrieving the latest schemna information:
+Users can specify a URL for retrieving the latest schemna information with `<confluent_registry>`:
 
-e.g.) `http(s)://[server fqdn]:[port]/subjects/[a great user's subject]/`
+e.g.)
+```
+  <confluent_registry>
+    url http://[confluent registry server ip]:[port]/
+    subject your-awesome-subject
+    # schema_key schema
+    # schema_version 1
+  </confluent_registry>
+```
 
 For example, when specifying the following configuration:
 
 ```
 <parse>
   @type avro
-  schema_registery_with_subject_url http://localhost:8081/subjects/persons-avro-value/
+  <confluent_registry>
+    url http://localhost:8081/
+    subject persons-avro-value
+    # schema_key schema
+    # schema_version 1
+  </confluent_registry>
 ```
 
-Then the parser plugin calls `GET http://localhost:8081/subjects/persons-avro-value/versions/` to retrive the registered schema versions and then calls `GET GET http://localhost:8081/subjects/persons-avro-value/versions/<the latest schema version>`.
+Then the parser plugin calls `GET http://localhost:8081/subjects/persons-avro-value/versions/latest` to retrive the registered schema versions. And when parsing failure occurred, this plugin will call `GET http://localhost:8081/schemas/ids/<schema id which is obtained from the second record on avro schema>`.
+
+If you use this plugin to parse confluent schema, please specify `use_confluent_schema` as `true`.
+
+This is because, confluent avro schema uses the following structure:
+
+MAGIC_BYTE | schema_id | record
+----------:|:---------:|:---------------
+ 1byte     |  4bytes   | record contents
+
+When specifying `<confluent_registry>` section on configuration, this plugin will skip to read the first 5-bytes automatically and parse `schema_id` from there.
 
 ## Copyright
 

data/{fluent-plugin-avro.gemspec → fluent-plugin-parser-avro.gemspec}

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-parser-avro"
-  spec.version = "0.1.0"
+  spec.version = "0.2.0"
   spec.authors = ["Hiroshi Hatake"]
   spec.email   = ["cosmo0920.wp@gmail.com"]
 

data/lib/fluent/plugin/confluent_avro_schema_registry.rb

@@ -0,0 +1,49 @@
+#
+# Copyright 2020- Hiroshi Hatake
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "net/http"
+require "uri"
+
+module Fluent
+  module Plugin
+    class ConfluentAvroSchemaRegistry
+      def initialize(registry_url)
+        @registry_url = registry_url
+      end
+
+      def subject_version(subject, schema_key, version = "latest")
+        registry_uri = URI.parse(@registry_url)
+        registry_uri_with_versions = URI.join(registry_uri, "/subjects/#{subject}/versions/#{version}")
+        response = Net::HTTP.get_response(registry_uri_with_versions)
+        if schema_key.nil?
+          response.body
+        else
+          Yajl.load(response.body)[schema_key]
+        end
+      end
+
+      def schema_with_id(schema_id, schema_key)
+        registry_uri = URI.parse(@registry_url)
+        registry_uri_with_ids = URI.join(registry_uri, "/schemas/ids/#{schema_id}")
+        response = Net::HTTP.get_response(registry_uri_with_ids)
+        if schema_key.nil?
+          response.body
+        else
+          Yajl.load(response.body)[schema_key]
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_avro.rb

@@ -18,21 +18,30 @@ require "net/http"
 require "stringio"
 require "uri"
 require "fluent/plugin/parser"
+require_relative "./confluent_avro_schema_registry"
 
 module Fluent
   module Plugin
     class AvroParser < Fluent::Plugin::Parser
       Fluent::Plugin.register_parser("avro", self)
 
+      MAGIC_BYTE = [0].pack("C").freeze
+
       config_param :schema_file, :string, :default => nil
       config_param :schema_json, :string, :default => nil
       config_param :schema_url, :string, :default => nil
-      config_param :schema_registery_with_subject_url, :string, :default => nil
       config_param :schema_url_key, :string, :default => nil
       config_param :writers_schema_file, :string, :default => nil
       config_param :writers_schema_json, :string, :default => nil
       config_param :readers_schema_file, :string, :default => nil
       config_param :readers_schema_json, :string, :default => nil
+      config_param :use_confluent_schema, :bool, :default => true
+      config_section :confluent_registry, param_name: :avro_registry, required: false, multi: false do
+        config_param :url, :string
+        config_param :subject, :string
+        config_param :schema_key, :string, :default => "schema"
+        config_param :schema_version, :string, :default => "latest"
+      end
 
       def configure(conf)
         super
@@ -60,18 +69,20 @@ module Fluent
           @writers_schema = Avro::Schema.parse(@writers_raw_schema)
           @readers_schema = Avro::Schema.parse(@readers_raw_schema)
           @reader = Avro::IO::DatumReader.new(@writers_schema, @readers_schema)
+        elsif @avro_registry
+          @confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url)
+          @raw_schema = @confluent_registry.subject_version(@avro_registry.subject,
+                                                            @avro_registry.schema_key,
+                                                            @avro_registry.schema_version)
+          @schema = Avro::Schema.parse(@raw_schema)
+          @reader = Avro::IO::DatumReader.new(@schema)
         else
-          unless [@schema_json, @schema_file, @schema_url, @schema_registery_with_subject_url].compact.size == 1
+          unless [@schema_json, @schema_file, @schema_url].compact.size == 1
             raise Fluent::ConfigError, "schema_json, schema_file, or schema_url is required, but they cannot specify at the same time!"
           end
-          if @schema_registery_with_subject_url && !@schema_registery_with_subject_url.end_with?("/")
-            raise Fluent::ConfigError, "schema_registery_with_subject_url must contain the trailing slash('/')."
-          end
 
           @raw_schema = if @schema_file
                           File.read(@schema_file)
-                        elsif @schema_registery_with_subject_url
-                          fetch_latest_schema(@schema_registery_with_subject_url, @schema_url_key)
                         elsif @schema_url
                           fetch_schema(@schema_url, @schema_url_key)
                         elsif @schema_json
@@ -91,25 +102,55 @@ module Fluent
         buffer = StringIO.new(data)
         decoder = Avro::IO::BinaryDecoder.new(buffer)
         begin
+          if @use_confluent_schema || @avro_registry
+            # When using confluent avro schema, record is formatted as follows:
+            #
+            # MAGIC_BYTE | schema_id | record
+            # ----------:|:---------:|:---------------
+            #  1byte     |  4bytes   | record contents
+            magic_byte = decoder.read(1)
+
+            if magic_byte != MAGIC_BYTE
+              raise "The first byte should be magic byte but got {magic_byte.inspect}"
+            end
+            schema_id = decoder.read(4).unpack("N").first
+          end
           decoded_data = @reader.read(decoder)
           time, record = convert_values(parse_time(decoded_data), decoded_data)
           yield time, record
-        rescue => e
-          raise e if @schema_url.nil? or @schema_registery_with_subject_url.nil?
+        rescue EOFError, RuntimeError => e
+          raise e unless [@schema_url, @avro_registry].compact.size == 1
           begin
             new_raw_schema = if @schema_url
                                fetch_schema(@schema_url, @schema_url_key)
-                             elsif @schema_registery_with_subject_url
-                               fetch_latest_schema(@schema_registery_with_subject_url, @schema_url_key)
+                             elsif @avro_registry
+                               @confluent_registry.schema_with_id(schema_id,
+                                                                  @avro_registry.schema_key)
                              end
             new_schema = Avro::Schema.parse(new_raw_schema)
-            is_changed = (new_raw_schena_== @raw_schema)
+            is_changed = (new_raw_schema != @raw_schema)
             @raw_schema = new_raw_schema
-            @schame = new_schema
-          rescue
+            @schema = new_schema
+          rescue EOFError, RuntimeError
             # Do nothing.
           end
           if is_changed
+            buffer = StringIO.new(data)
+            decoder = Avro::IO::BinaryDecoder.new(buffer)
+            if @use_confluent_schema || @avro_registry
+              # When using confluent avro schema, record is formatted as follows:
+              #
+              # MAGIC_BYTE | schema_id | record
+              # ----------:|:---------:|:---------------
+              #  1byte     |  4bytes   | record contents
+              magic_byte = decoder.read(1)
+
+              if magic_byte != MAGIC_BYTE
+                raise "The first byte should be magic byte but got {magic_byte.inspect}"
+              end
+              schema_id = decoder.read(4).unpack("N").first
+            end
+            @reader = Avro::IO::DatumReader.new(@schema)
             decoded_data = @reader.read(decoder)
             time, record = convert_values(parse_time(decoded_data), decoded_data)
             yield time, record
@@ -119,24 +160,6 @@ module Fluent
         end
       end
 
-      def fetch_schema_versions(base_uri_with_versions)
-        versions_response = Net::HTTP.get_response(base_uri_with_versions)
-        Yajl.load(versions_response.body)
-      end
-
-      def fetch_latest_schema(base_url, schema_key)
-        base_uri = URI.parse(base_url)
-        base_uri_with_versions = URI.join(base_uri, "versions/")
-        versions = fetch_schema_versions(base_uri_with_versions)
-        uri = URI.join(base_uri_with_versions, versions.last.to_s)
-        response = Net::HTTP.get_response(uri)
-        if schema_key.nil?
-          response.body
-        else
-          Yajl.load(response.body)[schema_key]
-        end
-      end
-
       def fetch_schema(url, schema_key)
         uri = URI.parse(url)
         response = Net::HTTP.get_response(uri)

data/test/data/schema-persions-value-1.avsc

@@ -0,0 +1 @@
+{"schema":"{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"com.ippontech.kafkatutorials\",\"fields\":[{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"},{\"name\":\"birthDate\",\"type\":\"long\"}]}"}

data/test/data/schema-persions-value-21.avsc

@@ -0,0 +1 @@
+{"schema":"{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"com.ippontech.kafkatutorials\",\"fields\":[{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"},{\"name\":\"birthDate\",\"type\":\"long\"},{\"name\":\"verified\",\"type\":\"boolean\",\"default\":false}]}"}

data/test/data/schema-persions-value-41.avsc

@@ -0,0 +1 @@
+{"schema":"{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"com.ippontech.kafkatutorials\",\"fields\":[{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"},{\"name\":\"birthDate\",\"type\":\"long\"},{\"name\":\"verified\",\"type\":\"boolean\"}]}"}

data/test/data/schema-persions-value-42.avsc

@@ -0,0 +1 @@
+{"schema":"{\"type\":\"record\",\"name\":\"Person\",\"namespace\":\"com.ippontech.kafkatutorials\",\"fields\":[{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"},{\"name\":\"birthDate\",\"type\":\"long\"},{\"name\":\"verified\",\"type\":[\"boolean\",\"null\"],\"default\":false}]}"}

data/test/plugin/test_parser_avro.rb

@@ -76,74 +76,94 @@ class AvroParserTest < Test::Unit::TestCase
     }
   EOC
 
-  def test_parse
+  data("use_confluent_schema" => true,
+       "plain"                => false)
+  def test_parse(data)
+    config = data
     conf = {
-      'schema_json' => SCHEMA
+      'schema_json' => SCHEMA,
+      'use_confluent_schema' => config,
     }
     d = create_driver(conf)
     datum = {"username" => "foo", "age" => 42, "verified" => true}
-    encoded = encode_datum(datum, SCHEMA)
+    encoded = encode_datum(datum, SCHEMA, config)
     d.instance.parse(encoded) do |_time, record|
       assert_equal datum, record
     end
 
     datum = {"username" => "baz", "age" => 34}
-    encoded = encode_datum(datum, SCHEMA)
+    encoded = encode_datum(datum, SCHEMA, config)
     d.instance.parse(encoded) do |_time, record|
       assert_equal datum.merge("verified" => nil), record
     end
   end
 
-  def test_parse_with_avro_schema
+  data("use_confluent_schema" => true,
+       "plain"                => false)
+  def test_parse_with_avro_schema(data)
+    config = data
     conf = {
-      'schema_file' => File.join(__dir__, "..", "data", "user.avsc")
+      'schema_file' => File.join(__dir__, "..", "data", "user.avsc"),
+      'use_confluent_schema' => config,
     }
     d = create_driver(conf)
     datum = {"username" => "foo", "age" => 42, "verified" => true}
-    encoded = encode_datum(datum, SCHEMA)
+    encoded = encode_datum(datum, SCHEMA, config)
     d.instance.parse(encoded) do |_time, record|
       assert_equal datum, record
     end
 
     datum = {"username" => "baz", "age" => 34}
-    encoded = encode_datum(datum, SCHEMA)
+    encoded = encode_datum(datum, SCHEMA, config)
     d.instance.parse(encoded) do |_time, record|
       assert_equal datum.merge("verified" => nil), record
     end
   end
 
-  def test_parse_with_readers_and_writers_schema
+  data("use_confluent_schema" => true,
+       "plain"                => false)
+  def test_parse_with_readers_and_writers_schema(data)
+    config = data
     conf = {
       'writers_schema_json' => SCHEMA,
       'readers_schema_json' => READERS_SCHEMA,
+      'use_confluent_schema' => config,
     }
     d = create_driver(conf)
     datum = {"username" => "foo", "age" => 42, "verified" => true}
-    encoded = encode_datum(datum, SCHEMA)
+    encoded = encode_datum(datum, SCHEMA, config)
     d.instance.parse(encoded) do |_time, record|
       datum.delete("verified")
       assert_equal datum, record
     end
   end
 
-  def test_parse_with_readers_and_writers_schema_files
+  data("use_confluent_schema" => true,
+       "plain"                => false)
+  def test_parse_with_readers_and_writers_schema_files(data)
+    config = data
     conf = {
       'writers_schema_file' => File.join(__dir__, "..", "data", "writer_user.avsc"),
       'readers_schema_file' => File.join(__dir__, "..", "data", "reader_user.avsc"),
+      'use_confluent_schema' => config,
     }
     d = create_driver(conf)
     datum = {"username" => "foo", "age" => 42, "verified" => true}
-    encoded = encode_datum(datum, SCHEMA)
+    encoded = encode_datum(datum, SCHEMA, config)
     d.instance.parse(encoded) do |_time, record|
       datum.delete("verified")
       assert_equal datum, record
     end
   end
 
-  def test_parse_with_complex_schema
+  data("use_confluent_schema" => true,
+       "plain"                => false)
+  def test_parse_with_complex_schema(data)
+    config = data
     conf = {
       'schema_json' => COMPLEX_SCHEMA,
-      'time_key' => 'time'
+      'time_key' => 'time',
+      'use_confluent_schema' => config,
     }
     d = create_driver(conf)
     time_str = "2020-09-25 15:08:09.082113 +0900"
@@ -162,7 +182,7 @@ class AvroParserTest < Test::Unit::TestCase
       }
     }
 
-    encoded = encode_datum(datum, COMPLEX_SCHEMA)
+    encoded = encode_datum(datum, COMPLEX_SCHEMA, config)
     d.instance.parse(encoded) do |time, record|
       assert_equal Time.parse(time_str).to_r, time.to_r
       datum.delete("time")
@@ -185,6 +205,22 @@ class AvroParserTest < Test::Unit::TestCase
           res.status = 200
           res.body = 'running'
         end
+        server.mount_proc("/schemas/ids") do |req, res|
+          req.path =~ /^\/schemas\/ids\/([^\/]*)$/
+          version = $1
+          @got.push({
+            version: version,
+          })
+          if version == "1"
+            res.body = File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc"))
+          elsif version == "21"
+            res.body = File.read(File.join(__dir__, "..", "data", "schema-persions-value-21.avsc"))
+          elsif version == "41"
+            res.body = File.read(File.join(__dir__, "..", "data", "schema-persions-value-41.avsc"))
+          elsif version == "42"
+            res.body = File.read(File.join(__dir__, "..", "data", "schema-persions-value-42.avsc"))
+          end
+        end
         server.mount_proc("/subjects") do |req, res|
           req.path =~ /^\/subjects\/([^\/]*)\/([^\/]*)\/(.*)$/
           avro_registered_name = $1
@@ -204,6 +240,8 @@ class AvroParserTest < Test::Unit::TestCase
             res.body = File.read(File.join(__dir__, "..", "data", "persons-avro-value3.avsc"))
           elsif version == "4"
             res.body = File.read(File.join(__dir__, "..", "data", "persons-avro-value4.avsc"))
+          elsif version == "latest"
+            res.body = File.read(File.join(__dir__, "..", "data", "persons-avro-value4.avsc"))
           end
         end
         server.start
@@ -318,63 +356,109 @@ class AvroParserTest < Test::Unit::TestCase
       assert_equal 4, @got.size
       assert_equal 'persons-avro-value', @got[3][:registered_name]
       assert_equal '3', @got[3][:version]
+
+      assert_equal '200', client.request_get('/schemas/ids/1').code
+      assert_equal 5, @got.size
+      assert_nil @got[4][:registered_name]
+      assert_equal '1', @got[4][:version]
+
+      assert_equal '200', client.request_get('/schemas/ids/21').code
+      assert_equal 6, @got.size
+      assert_nil @got[5][:registered_name]
+      assert_equal '21', @got[5][:version]
+
+      assert_equal '200', client.request_get('/schemas/ids/41').code
+      assert_equal 7, @got.size
+      assert_nil @got[6][:registered_name]
+      assert_equal '41', @got[6][:version]
+
+      assert_equal '200', client.request_get('/schemas/ids/42').code
+      assert_equal 8, @got.size
+      assert_nil @got[7][:registered_name]
+      assert_equal '42', @got[7][:version]
     end
 
-    def test_schema_url
+    data("use_confluent_schema" => true,
+         "plain"                => false)
+    def test_schema_url(data)
+      config = data
       conf = {
         'schema_url' => "http://localhost:8081/subjects/persons-avro-value/versions/1",
-        'schema_url_key' => 'schema'
+        'schema_url_key' => 'schema',
+        'use_confluent_schema' => config,
       }
       d = create_driver(conf)
       datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
-      encoded = encode_datum(datum, REMOTE_SCHEMA)
+      encoded = encode_datum(datum, REMOTE_SCHEMA, config)
       d.instance.parse(encoded) do |_time, record|
         assert_equal datum, record
       end
     end
 
-    def test_schema_url_with_version2
+    data("use_confluent_schema" => true,
+         "plain"                => false)
+    def test_schema_url_with_version2(data)
+      config = data
       conf = {
         'schema_url' => "http://localhost:8081/subjects/persons-avro-value/versions/2",
-        'schema_url_key' => 'schema'
+        'schema_url_key' => 'schema',
+        'use_confluent_schema' => config,
       }
       d = create_driver(conf)
       datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
-      encoded = encode_datum(datum, REMOTE_SCHEMA2)
+      encoded = encode_datum(datum, REMOTE_SCHEMA2, config)
       d.instance.parse(encoded) do |_time, record|
         assert_equal datum.merge("verified" => false), record
       end
     end
 
-    def test_schema_registery_with_subject_url
-      conf = {
-        'schema_registery_with_subject_url' => "http://localhost:8081/subjects/persons-avro-value/",
-        'schema_url_key' => 'schema'
-      }
+    def test_confluent_registry_with_schema_version
+      conf = Fluent::Config::Element.new(
+        '', '', {'@type' => 'avro'}, [
+          Fluent::Config::Element.new('confluent_registry', '', {
+                                        'url' => 'http://localhost:8081',
+                                        'subject' => 'persons-avro-value',
+                                        'schema_key' => 'schema',
+                                        'schema_version' => '1',
+                                      }, [])
+        ])
       d = create_driver(conf)
       datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
-      encoded = encode_datum(datum, REMOTE_SCHEMA2)
+      schema = Yajl.load(File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc")))
+      encoded = encode_datum(datum, schema.fetch("schema"), true, 1)
       d.instance.parse(encoded) do |_time, record|
-        assert_equal datum.merge("verified" => nil), record
+        assert_equal datum, record
       end
     end
 
-    def test_schema_registery_with_invalid_subject_url
-      conf = {
-        'schema_registery_with_subject_url' => "http://localhost:8081/subjects/persons-avro-value",
-        'schema_url_key' => 'schema'
-      }
-      assert_raise(Fluent::ConfigError) do
-        create_driver(conf)
+    def test_confluent_registry_with_fallback
+      conf = Fluent::Config::Element.new(
+        '', '', {'@type' => 'avro'}, [
+          Fluent::Config::Element.new('confluent_registry', '', {
+                                        'url' => 'http://localhost:8081',
+                                        'subject' => 'persons-avro-value',
+                                        'schema_key' => 'schema',
+                                      }, [])
+        ])
+      d = create_driver(conf)
+      datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
+      schema = Yajl.load(File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc")))
+      encoded = encode_datum(datum, schema.fetch("schema"), true, 1)
+      d.instance.parse(encoded) do |_time, record|
+        assert_equal datum, record
       end
     end
   end
 
   private
 
-  def encode_datum(datum, string_schema)
+  def encode_datum(datum, string_schema, use_confluent_schema = true, schema_id = 1)
     buffer = StringIO.new
     encoder = Avro::IO::BinaryEncoder.new(buffer)
+    if use_confluent_schema
+      encoder.write(Fluent::Plugin::AvroParser::MAGIC_BYTE)
+      encoder.write([schema_id].pack("N"))
+    end
     schema = Avro::Schema.parse(string_schema)
     writer = Avro::IO::DatumWriter.new(schema)
     writer.write(datum, encoder)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser-avro
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-29 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -100,13 +100,18 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- fluent-plugin-avro.gemspec
+- fluent-plugin-parser-avro.gemspec
+- lib/fluent/plugin/confluent_avro_schema_registry.rb
 - lib/fluent/plugin/parser_avro.rb
 - test/data/persons-avro-value.avsc
 - test/data/persons-avro-value2.avsc
 - test/data/persons-avro-value3.avsc
 - test/data/persons-avro-value4.avsc
 - test/data/reader_user.avsc
+- test/data/schema-persions-value-1.avsc
+- test/data/schema-persions-value-21.avsc
+- test/data/schema-persions-value-41.avsc
+- test/data/schema-persions-value-42.avsc
 - test/data/user.avsc
 - test/data/writer_user.avsc
 - test/helper.rb
@@ -140,6 +145,10 @@ test_files:
 - test/data/persons-avro-value3.avsc
 - test/data/persons-avro-value4.avsc
 - test/data/reader_user.avsc
+- test/data/schema-persions-value-1.avsc
+- test/data/schema-persions-value-21.avsc
+- test/data/schema-persions-value-41.avsc
+- test/data/schema-persions-value-42.avsc
 - test/data/user.avsc
 - test/data/writer_user.avsc
 - test/helper.rb