fluent-plugin-avro 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +9 -1
- data/fluent-plugin-avro.gemspec +1 -1
- data/lib/fluent/plugin/formatter_avro.rb +46 -4
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ab02eab8807cb85e002d461872fda200f2737d9
|
4
|
+
data.tar.gz: dde713066808be3f74935cf978a02abc5a492a9d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 69f61e2430613a3874ca7a87f74cb8976b7c3a29523878934dea8d85f7cf32183b5b27a0140a667ea7ea23a5c97092c7430883e84d45a363f75aa0a07324170b
|
7
|
+
data.tar.gz: 83d3d59940387324ce852d7c69c4f713f86806241d785a8b3f1456a1bc39755de3a78c46a5a9c66977db6f0c852497e838acf7513adb1d643ba3ae60bf481217
|
data/README.md
CHANGED
@@ -6,12 +6,14 @@ fluent-plugin-avro provides a formatter plugin for Fluentd.
|
|
6
6
|
|
7
7
|
## Configurations
|
8
8
|
|
9
|
-
Either `schema_file` or `
|
9
|
+
Either `schema_file`, `schema_json`, or `schema_url` is required.
|
10
10
|
|
11
11
|
| Name | Description |
|
12
12
|
| ---- | ----------- |
|
13
13
|
| `schema_file` | filename of Avro schema |
|
14
14
|
| `schema_json` | JSON representation of Avro schema |
|
15
|
+
| `schema_url` | URL to JSON representation of Avro schema |
|
16
|
+
| `schema_url_key` | JSON key under response body of where the JSON representation of Avro schema is |
|
15
17
|
|
16
18
|
### Example
|
17
19
|
|
@@ -26,6 +28,12 @@ Either `schema_file` or `schema_json` is required.
|
|
26
28
|
|
27
29
|
## You can use schema_json instead of schema_file
|
28
30
|
# schema_json {"type":"record","name":"example","namespace":"org.example","fields":[{"name":"message","type":"string"}]}
|
31
|
+
## You can also use schema_url to fetch from a schema registry
|
32
|
+
# schema_url http://localhost:8081/subjects/prod/versions/latest
|
33
|
+
# If your URL adds metadata around the schema, schema_url_key can be used to pull that out. If your response body was like this:
|
34
|
+
# {"subject":"prod","version":2,"id":2,"schema":"{\"type\":\"record\",\"name\":\"example\",\"namespace\":\"org.example\",\"fields\":[{\"name\":\"message\",\"type\":\"string\"}]}"}
|
35
|
+
# Then the below line would extract that out
|
36
|
+
# schema_url_key schema
|
29
37
|
</match>
|
30
38
|
```
|
31
39
|
|
data/fluent-plugin-avro.gemspec
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
require 'avro'
|
2
2
|
require 'fluent/plugin/formatter'
|
3
|
+
require 'net/http'
|
4
|
+
require 'uri'
|
5
|
+
require 'json'
|
3
6
|
|
4
7
|
module Fluent
|
5
8
|
module Plugin
|
@@ -8,15 +11,20 @@ module Fluent
|
|
8
11
|
|
9
12
|
config_param :schema_file, :string, :default => nil
|
10
13
|
config_param :schema_json, :string, :default => nil
|
14
|
+
config_param :schema_url, :string, :default => nil
|
15
|
+
config_param :schema_url_key, :string, :default => nil
|
11
16
|
|
12
17
|
def configure(conf)
|
13
18
|
super
|
14
|
-
if not (@schema_json.nil?
|
15
|
-
raise Fluent::ConfigError, 'schema_json or
|
19
|
+
if not ((@schema_json.nil? ? 0 : 1) + (@schema_file.nil? ? 0 : 1) + (@schema_url.nil? ? 0 : 1) == 1) then
|
20
|
+
raise Fluent::ConfigError, 'schema_json, schema_file, or schema_url is required, but not multiple!'
|
16
21
|
end
|
17
|
-
if @schema_json.nil? then
|
22
|
+
if (@schema_json.nil? && !@schema_file.nil?) then
|
18
23
|
@schema_json = File.read(@schema_file)
|
19
24
|
end
|
25
|
+
if (@schema_json.nil? && !@schema_url.nil?) then
|
26
|
+
@schema_json = fetch_schema(@schema_url,@schema_url_key)
|
27
|
+
end
|
20
28
|
@schema = Avro::Schema.parse(@schema_json)
|
21
29
|
@writer = Avro::IO::DatumWriter.new(@schema)
|
22
30
|
end
|
@@ -24,9 +32,43 @@ module Fluent
|
|
24
32
|
def format(tag, time, record)
|
25
33
|
buffer = StringIO.new
|
26
34
|
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
27
|
-
|
35
|
+
begin
|
36
|
+
@writer.write(record, encoder)
|
37
|
+
rescue => e
|
38
|
+
raise e if schema_url.nil?
|
39
|
+
schema_changed = false
|
40
|
+
begin
|
41
|
+
new_schema_json = fetch_schema(@schema_url,@schema_url_key)
|
42
|
+
new_schema = Avro::Schema.parse(@schema_json)
|
43
|
+
schema_changed = (new_schema_json == @schema_json)
|
44
|
+
@schema_json = new_schema_json
|
45
|
+
@schema = new_schema
|
46
|
+
rescue
|
47
|
+
end
|
48
|
+
if schema_changed then
|
49
|
+
@writer = Avro::IO::DatumWriter.new(@schema)
|
50
|
+
@writer.write(record, encoder)
|
51
|
+
else
|
52
|
+
raise e
|
53
|
+
end
|
54
|
+
end
|
28
55
|
buffer.string
|
29
56
|
end
|
57
|
+
|
58
|
+
def fetch_url(url)
|
59
|
+
uri = URI.parse(url)
|
60
|
+
response = Net::HTTP.get_response uri
|
61
|
+
response.body
|
62
|
+
end
|
63
|
+
|
64
|
+
def fetch_schema(url,schema_key)
|
65
|
+
response_body = fetch_url(url)
|
66
|
+
if schema_key.nil? then
|
67
|
+
return response_body
|
68
|
+
else
|
69
|
+
return JSON.parse(response_body)[schema_key]
|
70
|
+
end
|
71
|
+
end
|
30
72
|
end
|
31
73
|
end
|
32
74
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shun Takebayashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-08-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro
|
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
version: '0'
|
122
122
|
requirements: []
|
123
123
|
rubyforge_project:
|
124
|
-
rubygems_version: 2.5
|
124
|
+
rubygems_version: 2.4.5
|
125
125
|
signing_key:
|
126
126
|
specification_version: 4
|
127
127
|
summary: Avro formatter plugin for Fluentd
|