logstash-codec-avro 3.3.1-java → 3.4.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +12 -0
- data/lib/logstash/codecs/avro.rb +18 -2
- data/logstash-codec-avro.gemspec +1 -1
- data/spec/codecs/avro_spec.rb +51 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6efff24db98db6b6bf7bdbd677b0d95ed3a17baf454e7f7dfc1fa1e250643021
|
4
|
+
data.tar.gz: 2db283db2f02ae5c71b711f073dcf3bafcb3e58e9ee89b291c40af24b83f69a6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e7dfc41f0ca4172e24d69d37dd73c5467efb2ae872cfb23b95e08d43c2caf9db9de47b5489a9c3d08c3c97a31c6f128e1d8d82cfbf1b4ab09a2dc144f5f57da
|
7
|
+
data.tar.gz: 3d7752ac2c947d2456908ff14df389cad64ea30a13ea4d6e38fb84e4fdc41bef7cee8f7d88a8e3c7fb3e4270df62ea27f5b3778cf9d080109119c6002049be40
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 3.4.0
|
2
|
+
- Add `encoding` option to select the encoding of Avro payload, could be `binary` or `base64` [#39](https://github.com/logstash-plugins/logstash-codec-avro/pull/39)
|
3
|
+
|
1
4
|
## 3.3.1
|
2
5
|
- Pin avro gem to 1.10.x, as 1.11+ requires ruby 2.6+ [#37](https://github.com/logstash-plugins/logstash-codec-avro/pull/37)
|
3
6
|
|
data/docs/index.asciidoc
CHANGED
@@ -81,6 +81,7 @@ output {
|
|
81
81
|
[cols="<,<,<",options="header",]
|
82
82
|
|=======================================================================
|
83
83
|
|Setting |Input type|Required
|
84
|
+
| <<plugins-{type}s-{plugin}-encoding>> | <<string,string>>, one of `["binary", "base64"]`|No
|
84
85
|
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
85
86
|
| <<plugins-{type}s-{plugin}-schema_uri>> |<<string,string>>|Yes
|
86
87
|
| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<boolean,boolean>>|No
|
@@ -99,6 +100,17 @@ output {
|
|
99
100
|
|
100
101
|
Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
|
101
102
|
|
103
|
+
[id="plugins-{type}s-{plugin}-encoding"]
|
104
|
+
===== `encoding`
|
105
|
+
|
106
|
+
* Value can be any of: `binary`, `base64`
|
107
|
+
* Default value is `base64`
|
108
|
+
|
109
|
+
Set encoding for Avro's payload.
|
110
|
+
Use `base64` (default) to indicate that this codec sends or expects to receive base64-encoded bytes.
|
111
|
+
|
112
|
+
Set this option to `binary` to indicate that this codec sends or expects to receive binary Avro data.
|
113
|
+
|
102
114
|
|
103
115
|
[id="plugins-{type}s-{plugin}-schema_uri"]
|
104
116
|
===== `schema_uri`
|
data/lib/logstash/codecs/avro.rb
CHANGED
@@ -59,6 +59,14 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
59
59
|
|
60
60
|
include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
|
61
61
|
|
62
|
+
BINARY_ENCODING = "binary".freeze
|
63
|
+
BASE64_ENCODING = "base64".freeze
|
64
|
+
|
65
|
+
# Set encoding for Avro's payload.
|
66
|
+
# Use `base64` (default) encoding to convert the raw binary bytes to a `base64` encoded string.
|
67
|
+
# Set this option to `binary` to use the plain binary bytes.
|
68
|
+
config :encoding, :validate => [BINARY_ENCODING, BASE64_ENCODING], :default => BASE64_ENCODING
|
69
|
+
|
62
70
|
# schema path to fetch the schema from.
|
63
71
|
# This can be a 'http' or 'file' scheme URI
|
64
72
|
# example:
|
@@ -92,7 +100,11 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
92
100
|
|
93
101
|
public
|
94
102
|
def decode(data)
|
95
|
-
|
103
|
+
if encoding == BASE64_ENCODING
|
104
|
+
datum = StringIO.new(Base64.strict_decode64(data)) rescue StringIO.new(data)
|
105
|
+
else
|
106
|
+
datum = StringIO.new(data)
|
107
|
+
end
|
96
108
|
decoder = Avro::IO::BinaryDecoder.new(datum)
|
97
109
|
datum_reader = Avro::IO::DatumReader.new(@schema)
|
98
110
|
event = targeted_event_factory.new_event(datum_reader.read(decoder))
|
@@ -113,6 +125,10 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
113
125
|
buffer = StringIO.new
|
114
126
|
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
115
127
|
dw.write(event.to_hash, encoder)
|
116
|
-
|
128
|
+
if encoding == BASE64_ENCODING
|
129
|
+
@on_event.call(event, Base64.strict_encode64(buffer.string))
|
130
|
+
else
|
131
|
+
@on_event.call(event, buffer.string)
|
132
|
+
end
|
117
133
|
end
|
118
134
|
end
|
data/logstash-codec-avro.gemspec
CHANGED
data/spec/codecs/avro_spec.rb
CHANGED
@@ -55,6 +55,35 @@ describe LogStash::Codecs::Avro, :ecs_compatibility_support, :aggregate_failures
|
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
58
|
+
context "with binary encoding" do
|
59
|
+
let (:avro_config) { super().merge('encoding' => 'binary') }
|
60
|
+
|
61
|
+
it "should return an LogStash::Event from raw and base64 encoded avro data" do
|
62
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
63
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
64
|
+
buffer = StringIO.new
|
65
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
66
|
+
dw.write(test_event.to_hash, encoder)
|
67
|
+
|
68
|
+
subject.decode(buffer.string) do |event|
|
69
|
+
expect(event).to be_a_kind_of(LogStash::Event)
|
70
|
+
expect(event.get("foo")).to eq(test_event.get("foo"))
|
71
|
+
expect(event.get("bar")).to eq(test_event.get("bar"))
|
72
|
+
expect(event.get('[event][original]')).to eq(buffer.string) if ecs_compatibility != :disabled
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
it "should raise an error if base64 encoded data is provided" do
|
77
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
78
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
79
|
+
buffer = StringIO.new
|
80
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
81
|
+
dw.write(test_event.to_hash, encoder)
|
82
|
+
|
83
|
+
expect {subject.decode(Base64.strict_encode64(buffer.string))}.to raise_error
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
58
87
|
context "#decode with tag_on_failure" do
|
59
88
|
let (:avro_config) {{ 'schema_uri' => '
|
60
89
|
{"type": "record", "name": "Test",
|
@@ -111,6 +140,28 @@ describe LogStash::Codecs::Avro, :ecs_compatibility_support, :aggregate_failures
|
|
111
140
|
insist {got_event}
|
112
141
|
end
|
113
142
|
|
143
|
+
context "with binary encoding" do
|
144
|
+
let (:avro_config) { super().merge('encoding' => 'binary') }
|
145
|
+
|
146
|
+
it "should return avro data from a LogStash::Event not base64 encoded" do
|
147
|
+
got_event = false
|
148
|
+
subject.on_event do |event, data|
|
149
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
150
|
+
datum = StringIO.new(data)
|
151
|
+
decoder = Avro::IO::BinaryDecoder.new(datum)
|
152
|
+
datum_reader = Avro::IO::DatumReader.new(schema)
|
153
|
+
record = datum_reader.read(decoder)
|
154
|
+
|
155
|
+
expect(event).to be_a_kind_of(LogStash::Event)
|
156
|
+
expect(event.get("foo")).to eq(test_event.get("foo"))
|
157
|
+
expect(event.get("bar")).to eq(test_event.get("bar"))
|
158
|
+
got_event = true
|
159
|
+
end
|
160
|
+
subject.encode(test_event)
|
161
|
+
expect(got_event).to be true
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
114
165
|
context "binary data" do
|
115
166
|
|
116
167
|
let (:avro_config) {{ 'schema_uri' => '{"namespace": "com.systems.test.data",
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.4.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|