fluent-plugin-parser-avro 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9633a316c1de1e4cb83b487d99d5a3814f1cde2b96d20633cfe728c584108761
-  data.tar.gz: 6438fe5e248c98540602183a5828a7dbc596fe5000c4993fd428ee9223ed3077
+  metadata.gz: b0329676a89e7150a2dbd1b276496d81585acf3a8eba105beacdfbe22260efbb
+  data.tar.gz: 7a32762b40f316238d3dc45a18741e14ffc307154b643936f766e0c69b979dfe
 SHA512:
-  metadata.gz: 8b8894c17aaa33916ef3601634d36139f2817db875eba2c5e552a0309f2c5f3ce6578526c5aad54244cd51ce36cf30bc04f64ce576fcbd4ccb89fc7726d4102d
-  data.tar.gz: a9dbc9e144b1e0f7f2b585a51c0519e6c2953d7258b5bd13fd8d6246754f9956bffcee43777378692ea4ef2d8997d8f6648653d8928e2345e70c32e48ab849a6
+  metadata.gz: 2a97b4a1dac117a0ff810de6f45bb11131782a13af5739accdea57741fdf00d12d786a17ca7e54f5072517995f813216129e4d682e452a25813e9ca4da875d02
+  data.tar.gz: fd63a06ae328e8f8fd48f8be53f86a62d4b1085df411a8acfdb9a62fc1c80b943b3ee4b6d4272c7df7ac260e74f03ed308e4efce07c3ab9bb97adb5569da3918

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '2.5', '2.6', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        ruby: [ '2.5', '2.6', '2.7' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/README.md

@@ -39,6 +39,8 @@ $ bundle
 * **readers_schema_json** (string) (optional): avro schema definition hash for readers definition.
 * **use_confluent_schema** (bool) (optional): Assume to use confluent schema. Confluent avro schema uses the first 5-bytes for magic byte (1 byte) and schema_id (4 bytes). This parameter specifies to skip reading the first 5-bytes or not.
   * Default value: `true`.
+* **api_key** (string) (optional): Set key for Basic authentication.
+* **api_secret** (string) (optional): Set secret for Basic authentication.
 
 ### \<confluent_registry\> section (optional) (single)
 
@@ -76,6 +78,7 @@ Confluent AVRO schema registry should respond with REST API.
 This plugin uses the following API:
 
 * [`GET /subjects/(string: subject)/versions/(versionId: version)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--subjects-(string-%20subject)-versions)
+* [`GET /schemas/ids/(int: id)`](https://docs.confluent.io/current/schema-registry/develop/api.html#get--schemas-ids-int-%20id)
 
 Users can specify a URL for retrieving the latest schemna information with `<confluent_registry>`:
 

data/fluent-plugin-parser-avro.gemspec

@@ -3,9 +3,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-parser-avro"
-  spec.version = "0.2.0"
-  spec.authors = ["Hiroshi Hatake"]
-  spec.email   = ["cosmo0920.wp@gmail.com"]
+  spec.version = "0.3.0"
+  spec.authors = ["Hiroshi Hatake", "Kentaro Hayashi"]
+  spec.email   = ["cosmo0920.wp@gmail.com", "kenhys@gmail.com"]
 
   spec.summary       = %q{Avro parser plugin for Fluentd}
   spec.description   = spec.summary

data/lib/fluent/plugin/confluent_avro_schema_registry.rb

@@ -18,15 +18,19 @@ require "uri"
 
 module Fluent
   module Plugin
+    class ConfluentAvroSchemaRegistryUnauthorizedError < StandardError; end
+
     class ConfluentAvroSchemaRegistry
-      def initialize(registry_url)
+      def initialize(registry_url, api_key=nil, api_secret=nil)
         @registry_url = registry_url
+        @api_key = api_key
+        @api_secret = api_secret
       end
 
       def subject_version(subject, schema_key, version = "latest")
         registry_uri = URI.parse(@registry_url)
         registry_uri_with_versions = URI.join(registry_uri, "/subjects/#{subject}/versions/#{version}")
-        response = Net::HTTP.get_response(registry_uri_with_versions)
+        response = get_response(registry_uri_with_versions)
         if schema_key.nil?
           response.body
         else
@@ -37,13 +41,31 @@ module Fluent
       def schema_with_id(schema_id, schema_key)
         registry_uri = URI.parse(@registry_url)
         registry_uri_with_ids = URI.join(registry_uri, "/schemas/ids/#{schema_id}")
-        response = Net::HTTP.get_response(registry_uri_with_ids)
+        response = get_response(registry_uri_with_ids)
         if schema_key.nil?
           response.body
         else
           Yajl.load(response.body)[schema_key]
         end
       end
+
+      def get_response(uri)
+        response = if @api_key and @api_secret
+                     Net::HTTP.start(uri.host, uri.port) do |http|
+                       request = Net::HTTP::Get.new(uri.path)
+                       request.basic_auth(@api_key, @api_secret)
+                       http.request(request)
+                     end
+                   else
+                     Net::HTTP.get_response(uri)
+                   end
+        if @api_key and @api_secret
+          if response.is_a?(Net::HTTPUnauthorized)
+            raise ConfluentAvroSchemaRegistryUnauthorizedError
+          end
+        end
+        response
+      end
     end
   end
 end

data/lib/fluent/plugin/parser_avro.rb

@@ -36,6 +36,8 @@ module Fluent
       config_param :readers_schema_file, :string, :default => nil
       config_param :readers_schema_json, :string, :default => nil
       config_param :use_confluent_schema, :bool, :default => true
+      config_param :api_key, :string, :default => nil
+      config_param :api_secret, :string, :default => nil
       config_section :confluent_registry, param_name: :avro_registry, required: false, multi: false do
         config_param :url, :string
         config_param :subject, :string
@@ -70,7 +72,7 @@ module Fluent
           @readers_schema = Avro::Schema.parse(@readers_raw_schema)
           @reader = Avro::IO::DatumReader.new(@writers_schema, @readers_schema)
         elsif @avro_registry
-          @confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url)
+          @confluent_registry = Fluent::Plugin::ConfluentAvroSchemaRegistry.new(@avro_registry.url, @api_key, @api_secret)
           @raw_schema = @confluent_registry.subject_version(@avro_registry.subject,
                                                             @avro_registry.schema_key,
                                                             @avro_registry.schema_version)
@@ -162,7 +164,15 @@ module Fluent
 
       def fetch_schema(url, schema_key)
         uri = URI.parse(url)
-        response = Net::HTTP.get_response(uri)
+        response = if @api_key and @api_secret
+                     Net::HTTP.start(uri.host, uri.port) do |http|
+                       request = Net::HTTP::Get.new(uri.path)
+                       request.basic_auth(@api_key, @api_secret)
+                       http.request(request)
+                     end
+                   else
+                     Net::HTTP.get_response(uri)
+                   end
         if schema_key.nil?
           response.body
         else

data/test/plugin/test_parser_avro.rb

@@ -450,6 +450,70 @@ class AvroParserTest < Test::Unit::TestCase
     end
   end
 
+  class AuthenticationTest < self
+    teardown do
+      @dummy_server_thread.kill
+      @dummy_server_thread.join
+    end
+
+    setup do
+      @got = []
+      @dummy_server_thread = Thread.new do
+        server = WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => AVRO_REGISTRY_PORT})
+        begin
+          htpasswd = WEBrick::HTTPAuth::Htpasswd.new('dot.htpasswd')
+          htpasswd.set_passwd(nil, "API_KEY", "API_SECRET")
+          authenticator = WEBrick::HTTPAuth::BasicAuth.new(:UserDB => htpasswd, :Realm => "")
+          server.mount_proc('/') do |req, res|
+            authenticator.authenticate(req, res)
+            schema = File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc"))
+            res.body = schema
+          end
+          server.start
+        ensure
+          server.shutdown
+        end
+      end
+    end
+
+    def test_authentication_failure
+      conf = Fluent::Config::Element.new(
+        '', '', {'@type' => 'avro',
+                 'api_key' => 'WRONG_KEY',
+                 'api_secret' => 'WRONG_SECRET'
+                }, [
+          Fluent::Config::Element.new('confluent_registry', '', {
+                                        'url' => 'http://localhost:8081',
+                                        'subject' => 'persons-avro-value',
+                                      }, [])
+        ])
+      assert_raise(Fluent::Plugin::ConfluentAvroSchemaRegistryUnauthorizedError) do
+        d = create_driver(conf)
+        d.instance.run
+      end
+    end
+
+    def test_with_authentication
+      conf = Fluent::Config::Element.new(
+        '', '', {'@type' => 'avro',
+                 'api_key' => 'API_KEY',
+                 'api_secret' => 'API_SECRET'
+                }, [
+          Fluent::Config::Element.new('confluent_registry', '', {
+                                        'url' => 'http://localhost:8081',
+                                        'subject' => 'persons-avro-value',
+                                      }, [])
+        ])
+      datum = {"firstName" => "Aleen","lastName" => "Terry","birthDate" => 159202477258}
+      schema = Yajl.load(File.read(File.join(__dir__, "..", "data", "schema-persions-value-1.avsc")))
+      encoded = encode_datum(datum, schema.fetch("schema"), true, 1)
+      d = create_driver(conf)
+      d.instance.parse(encoded) do |_time, record|
+        assert_equal datum, record
+      end
+    end
+  end
+
   private
 
   def encode_datum(datum, string_schema, use_confluent_schema = true, schema_id = 1)

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser-avro
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Hiroshi Hatake
+- Kentaro Hayashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-30 00:00:00.000000000 Z
+date: 2021-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -89,6 +90,7 @@ dependencies:
 description: Avro parser plugin for Fluentd
 email:
 - cosmo0920.wp@gmail.com
+- kenhys@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []