fluent-plugin-avro 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4fab5e7e69c6cce1bce0266dfbf9699d0af6185
-  data.tar.gz: afca739410951904efaa270017474838413c3efb
+  metadata.gz: 6ab02eab8807cb85e002d461872fda200f2737d9
+  data.tar.gz: dde713066808be3f74935cf978a02abc5a492a9d
 SHA512:
-  metadata.gz: 4e322d7c56b3b689ee4d6ea9ecbda32e4eb5969a839ac71510e2da27818f0eb7302fdca52de992cda7857fa9f0f2d8bfbe14368f6e61f3de4f5513048443b3e5
-  data.tar.gz: b2bcb51d60ac51dc1948a72fdb8f62ece2a2e448217880b055e442d0af4401dc15c375558b8c165badc1ec98c7d265c1f8c8c1de1cd805ec8beaeb499a013edc
+  metadata.gz: 69f61e2430613a3874ca7a87f74cb8976b7c3a29523878934dea8d85f7cf32183b5b27a0140a667ea7ea23a5c97092c7430883e84d45a363f75aa0a07324170b
+  data.tar.gz: 83d3d59940387324ce852d7c69c4f713f86806241d785a8b3f1456a1bc39755de3a78c46a5a9c66977db6f0c852497e838acf7513adb1d643ba3ae60bf481217

data/README.md

@@ -6,12 +6,14 @@ fluent-plugin-avro provides a formatter plugin for Fluentd.
 
 ## Configurations
 
-Either `schema_file` or `schema_json` is required.
+Either `schema_file`, `schema_json`, or `schema_url` is required.
 
 | Name | Description |
 | ---- | ----------- |
 | `schema_file` | filename of Avro schema |
 | `schema_json` | JSON representation of Avro schema |
+| `schema_url`  | URL to JSON representation of Avro schema |
+| `schema_url_key`  | JSON key under response body of where the JSON representation of Avro schema is |
 
 ### Example
 
@@ -26,6 +28,12 @@ Either `schema_file` or `schema_json` is required.
 
   ## You can use schema_json instead of schema_file
   # schema_json {"type":"record","name":"example","namespace":"org.example","fields":[{"name":"message","type":"string"}]}
+  ## You can also use schema_url to fetch from a schema registry
+  # schema_url http://localhost:8081/subjects/prod/versions/latest
+  # If your URL adds metadata around the schema, schema_url_key can be used to pull that out. If your response body was like this:
+  # {"subject":"prod","version":2,"id":2,"schema":"{\"type\":\"record\",\"name\":\"example\",\"namespace\":\"org.example\",\"fields\":[{\"name\":\"message\",\"type\":\"string\"}]}"}
+  # Then the below line would extract that out
+  # schema_url_key schema
 </match>
 ```
 

data/fluent-plugin-avro.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-avro"
-  spec.version       = "1.0.0"
+  spec.version       = "1.1.0"
   spec.authors       = ["Shun Takebayashi"]
   spec.email         = ["shun@takebayashi.asia"]
 

data/lib/fluent/plugin/formatter_avro.rb

@@ -1,5 +1,8 @@
 require 'avro'
 require 'fluent/plugin/formatter'
+require 'net/http'
+require 'uri'
+require 'json'
 
 module Fluent
   module Plugin
@@ -8,15 +11,20 @@ module Fluent
 
       config_param :schema_file, :string, :default => nil
       config_param :schema_json, :string, :default => nil
+      config_param :schema_url, :string, :default => nil
+      config_param :schema_url_key, :string, :default => nil
 
       def configure(conf)
         super
-        if not (@schema_json.nil? ^ @schema_file.nil?) then
-          raise Fluent::ConfigError, 'schema_json or schema_file (but not both) is required'
+        if not ((@schema_json.nil? ? 0 : 1) + (@schema_file.nil? ? 0 : 1) + (@schema_url.nil? ? 0 : 1) == 1) then
+          raise Fluent::ConfigError, 'schema_json, schema_file, or schema_url is required, but not multiple!'
         end
-        if @schema_json.nil? then
+        if (@schema_json.nil? && !@schema_file.nil?) then
           @schema_json = File.read(@schema_file)
         end
+        if (@schema_json.nil? && !@schema_url.nil?) then
+          @schema_json = fetch_schema(@schema_url,@schema_url_key)
+        end
         @schema = Avro::Schema.parse(@schema_json)
         @writer = Avro::IO::DatumWriter.new(@schema)
       end
@@ -24,9 +32,43 @@ module Fluent
       def format(tag, time, record)
         buffer = StringIO.new
         encoder = Avro::IO::BinaryEncoder.new(buffer)
-        @writer.write(record, encoder)
+        begin
+          @writer.write(record, encoder)
+        rescue => e
+          raise e if schema_url.nil?
+          schema_changed = false
+          begin
+            new_schema_json = fetch_schema(@schema_url,@schema_url_key)
+            new_schema = Avro::Schema.parse(@schema_json)
+            schema_changed = (new_schema_json == @schema_json)
+            @schema_json = new_schema_json
+            @schema = new_schema
+          rescue
+          end
+          if schema_changed then
+            @writer = Avro::IO::DatumWriter.new(@schema)
+            @writer.write(record, encoder)
+          else
+            raise e
+          end
+        end
         buffer.string
       end
+
+      def fetch_url(url)
+        uri = URI.parse(url)
+        response = Net::HTTP.get_response uri
+        response.body
+      end
+
+      def fetch_schema(url,schema_key)
+        response_body = fetch_url(url)
+        if schema_key.nil? then
+          return response_body
+        else
+          return JSON.parse(response_body)[schema_key]
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-avro
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Shun Takebayashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-17 00:00:00.000000000 Z
+date: 2019-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Avro formatter plugin for Fluentd