logstash-codec-avro 3.2.4-java → 3.4.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +1 -1
- data/docs/index.asciidoc +51 -0
- data/lib/logstash/codecs/avro.rb +48 -9
- data/logstash-codec-avro.gemspec +6 -3
- data/spec/codecs/avro_spec.rb +155 -67
- metadata +49 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6efff24db98db6b6bf7bdbd677b0d95ed3a17baf454e7f7dfc1fa1e250643021
|
4
|
+
data.tar.gz: 2db283db2f02ae5c71b711f073dcf3bafcb3e58e9ee89b291c40af24b83f69a6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e7dfc41f0ca4172e24d69d37dd73c5467efb2ae872cfb23b95e08d43c2caf9db9de47b5489a9c3d08c3c97a31c6f128e1d8d82cfbf1b4ab09a2dc144f5f57da
|
7
|
+
data.tar.gz: 3d7752ac2c947d2456908ff14df389cad64ea30a13ea4d6e38fb84e4fdc41bef7cee8f7d88a8e3c7fb3e4270df62ea27f5b3778cf9d080109119c6002049be40
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
## 3.4.0
|
2
|
+
- Add `encoding` option to select the encoding of Avro payload, could be `binary` or `base64` [#39](https://github.com/logstash-plugins/logstash-codec-avro/pull/39)
|
3
|
+
|
4
|
+
## 3.3.1
|
5
|
+
- Pin avro gem to 1.10.x, as 1.11+ requires ruby 2.6+ [#37](https://github.com/logstash-plugins/logstash-codec-avro/pull/37)
|
6
|
+
|
7
|
+
## 3.3.0
|
8
|
+
- Add ECS support. Add target option and event.original [#36](https://github.com/logstash-plugins/logstash-codec-avro/pull/36)
|
9
|
+
|
1
10
|
## 3.2.4
|
2
11
|
- [DOC] Add clarifications on partial deserialization [#35](https://github.com/logstash-plugins/logstash-codec-avro/pull/35)
|
3
12
|
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
[![Travis Build Status](https://travis-ci.
|
3
|
+
[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-codec-avro.svg)](https://travis-ci.com/logstash-plugins/logstash-codec-avro)
|
4
4
|
|
5
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
6
6
|
|
data/docs/index.asciidoc
CHANGED
@@ -26,6 +26,11 @@ This plugin is used to serialize Logstash events as
|
|
26
26
|
Avro datums, as well as deserializing Avro datums into
|
27
27
|
Logstash events.
|
28
28
|
|
29
|
+
[id="plugins-{type}s-{plugin}-ecs_metadata"]
|
30
|
+
==== Event Metadata and the Elastic Common Schema (ECS)
|
31
|
+
|
32
|
+
The plugin behaves the same regardless of ECS compatibility, except adding the original message to `[event][original]`.
|
33
|
+
|
29
34
|
==== Encoding
|
30
35
|
|
31
36
|
This codec is for serializing individual Logstash events
|
@@ -76,12 +81,37 @@ output {
|
|
76
81
|
[cols="<,<,<",options="header",]
|
77
82
|
|=======================================================================
|
78
83
|
|Setting |Input type|Required
|
84
|
+
| <<plugins-{type}s-{plugin}-encoding>> | <<string,string>>, one of `["binary", "base64"]`|No
|
85
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
79
86
|
| <<plugins-{type}s-{plugin}-schema_uri>> |<<string,string>>|Yes
|
80
87
|
| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<boolean,boolean>>|No
|
88
|
+
| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
|
81
89
|
|=======================================================================
|
82
90
|
|
83
91
|
|
84
92
|
|
93
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
94
|
+
===== `ecs_compatibility`
|
95
|
+
|
96
|
+
* Value type is <<string,string>>
|
97
|
+
* Supported values are:
|
98
|
+
** `disabled`: Avro data added at root level
|
99
|
+
** `v1`,`v8`: Elastic Common Schema compliant behavior (`[event][original]` is also added)
|
100
|
+
|
101
|
+
Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
|
102
|
+
|
103
|
+
[id="plugins-{type}s-{plugin}-encoding"]
|
104
|
+
===== `encoding`
|
105
|
+
|
106
|
+
* Value can be any of: `binary`, `base64`
|
107
|
+
* Default value is `base64`
|
108
|
+
|
109
|
+
Set encoding for Avro's payload.
|
110
|
+
Use `base64` (default) to indicate that this codec sends or expects to receive base64-encoded bytes.
|
111
|
+
|
112
|
+
Set this option to `binary` to indicate that this codec sends or expects to receive binary Avro data.
|
113
|
+
|
114
|
+
|
85
115
|
[id="plugins-{type}s-{plugin}-schema_uri"]
|
86
116
|
===== `schema_uri`
|
87
117
|
|
@@ -104,4 +134,25 @@ example:
|
|
104
134
|
|
105
135
|
tag events with `_avroparsefailure` when decode fails
|
106
136
|
|
137
|
+
[id="plugins-{type}s-{plugin}-target"]
|
138
|
+
===== `target`
|
139
|
+
|
140
|
+
* Value type is <<string,string>>
|
141
|
+
* There is no default value for this setting.
|
142
|
+
* This is only relevant when decode data into an event
|
107
143
|
|
144
|
+
Define the target field for placing the values. If this setting is not
|
145
|
+
set, the Avro data will be stored at the root (top level) of the event.
|
146
|
+
|
147
|
+
*Example*
|
148
|
+
[source,ruby]
|
149
|
+
----------------------------------
|
150
|
+
input {
|
151
|
+
kafka {
|
152
|
+
codec => avro {
|
153
|
+
schema_uri => "/tmp/schema.avsc"
|
154
|
+
target => "[document]"
|
155
|
+
}
|
156
|
+
}
|
157
|
+
}
|
158
|
+
----------------------------------
|
data/lib/logstash/codecs/avro.rb
CHANGED
@@ -6,17 +6,21 @@ require "logstash/codecs/base"
|
|
6
6
|
require "logstash/event"
|
7
7
|
require "logstash/timestamp"
|
8
8
|
require "logstash/util"
|
9
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
10
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
|
11
|
+
require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
|
12
|
+
require 'logstash/plugin_mixins/event_support/event_factory_adapter'
|
9
13
|
|
10
14
|
# Read serialized Avro records as Logstash events
|
11
15
|
#
|
12
|
-
# This plugin is used to serialize Logstash events as
|
13
|
-
# Avro datums, as well as deserializing Avro datums into
|
16
|
+
# This plugin is used to serialize Logstash events as
|
17
|
+
# Avro datums, as well as deserializing Avro datums into
|
14
18
|
# Logstash events.
|
15
19
|
#
|
16
20
|
# ==== Encoding
|
17
|
-
#
|
18
|
-
# This codec is for serializing individual Logstash events
|
19
|
-
# as Avro datums that are Avro binary blobs. It does not encode
|
21
|
+
#
|
22
|
+
# This codec is for serializing individual Logstash events
|
23
|
+
# as Avro datums that are Avro binary blobs. It does not encode
|
20
24
|
# Logstash events into an Avro file.
|
21
25
|
#
|
22
26
|
#
|
@@ -48,6 +52,20 @@ require "logstash/util"
|
|
48
52
|
class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
49
53
|
config_name "avro"
|
50
54
|
|
55
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
56
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
|
57
|
+
|
58
|
+
extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
|
59
|
+
|
60
|
+
include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
|
61
|
+
|
62
|
+
BINARY_ENCODING = "binary".freeze
|
63
|
+
BASE64_ENCODING = "base64".freeze
|
64
|
+
|
65
|
+
# Set encoding for Avro's payload.
|
66
|
+
# Use `base64` (default) encoding to convert the raw binary bytes to a `base64` encoded string.
|
67
|
+
# Set this option to `binary` to use the plain binary bytes.
|
68
|
+
config :encoding, :validate => [BINARY_ENCODING, BASE64_ENCODING], :default => BASE64_ENCODING
|
51
69
|
|
52
70
|
# schema path to fetch the schema from.
|
53
71
|
# This can be a 'http' or 'file' scheme URI
|
@@ -60,25 +78,42 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
60
78
|
# tag events with `_avroparsefailure` when decode fails
|
61
79
|
config :tag_on_failure, :validate => :boolean, :default => false
|
62
80
|
|
81
|
+
# Defines a target field for placing decoded fields.
|
82
|
+
# If this setting is omitted, data gets stored at the root (top level) of the event.
|
83
|
+
#
|
84
|
+
# NOTE: the target is only relevant while decoding data into a new event.
|
85
|
+
config :target, :validate => :field_reference
|
86
|
+
|
63
87
|
def open_and_read(uri_string)
|
64
88
|
open(uri_string).read
|
65
89
|
end
|
66
90
|
|
67
91
|
public
|
92
|
+
def initialize(*params)
|
93
|
+
super
|
94
|
+
@original_field = ecs_select[disabled: nil, v1: '[event][original]']
|
95
|
+
end
|
96
|
+
|
68
97
|
def register
|
69
98
|
@schema = Avro::Schema.parse(open_and_read(schema_uri))
|
70
99
|
end
|
71
100
|
|
72
101
|
public
|
73
102
|
def decode(data)
|
74
|
-
|
103
|
+
if encoding == BASE64_ENCODING
|
104
|
+
datum = StringIO.new(Base64.strict_decode64(data)) rescue StringIO.new(data)
|
105
|
+
else
|
106
|
+
datum = StringIO.new(data)
|
107
|
+
end
|
75
108
|
decoder = Avro::IO::BinaryDecoder.new(datum)
|
76
109
|
datum_reader = Avro::IO::DatumReader.new(@schema)
|
77
|
-
|
110
|
+
event = targeted_event_factory.new_event(datum_reader.read(decoder))
|
111
|
+
event.set(@original_field, data.dup.freeze) if @original_field
|
112
|
+
yield event
|
78
113
|
rescue => e
|
79
114
|
if tag_on_failure
|
80
115
|
@logger.error("Avro parse error, original data now in message field", :error => e)
|
81
|
-
yield
|
116
|
+
yield event_factory.new_event("message" => data, "tags" => ["_avroparsefailure"])
|
82
117
|
else
|
83
118
|
raise e
|
84
119
|
end
|
@@ -90,6 +125,10 @@ class LogStash::Codecs::Avro < LogStash::Codecs::Base
|
|
90
125
|
buffer = StringIO.new
|
91
126
|
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
92
127
|
dw.write(event.to_hash, encoder)
|
93
|
-
|
128
|
+
if encoding == BASE64_ENCODING
|
129
|
+
@on_event.call(event, Base64.strict_encode64(buffer.string))
|
130
|
+
else
|
131
|
+
@on_event.call(event, buffer.string)
|
132
|
+
end
|
94
133
|
end
|
95
134
|
end
|
data/logstash-codec-avro.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-avro'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.4.0'
|
5
5
|
s.platform = 'java'
|
6
6
|
s.licenses = ['Apache-2.0']
|
7
7
|
s.summary = "Reads serialized Avro records as Logstash events"
|
@@ -22,10 +22,13 @@ Gem::Specification.new do |s|
|
|
22
22
|
|
23
23
|
# Gem dependencies
|
24
24
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
25
|
-
|
26
|
-
s.add_runtime_dependency
|
25
|
+
s.add_runtime_dependency "avro", "~> 1.10.2" #(Apache 2.0 license)
|
26
|
+
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
|
27
|
+
s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
|
28
|
+
s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
|
27
29
|
|
28
30
|
s.add_development_dependency 'logstash-devutils'
|
29
31
|
s.add_development_dependency 'insist'
|
32
|
+
|
30
33
|
end
|
31
34
|
|
data/spec/codecs/avro_spec.rb
CHANGED
@@ -5,80 +5,166 @@ require 'avro'
|
|
5
5
|
require 'base64'
|
6
6
|
require 'logstash/codecs/avro'
|
7
7
|
require 'logstash/event'
|
8
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
|
8
9
|
|
9
|
-
describe LogStash::Codecs::Avro do
|
10
|
+
describe LogStash::Codecs::Avro, :ecs_compatibility_support, :aggregate_failures do
|
10
11
|
|
11
|
-
|
12
|
-
|
12
|
+
ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
|
13
|
+
before(:each) do
|
14
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
15
|
+
end
|
16
|
+
|
17
|
+
context "non binary data" do
|
18
|
+
let (:avro_config) {{ 'schema_uri' => '
|
13
19
|
{"type": "record", "name": "Test",
|
14
20
|
"fields": [{"name": "foo", "type": ["null", "string"]},
|
15
21
|
{"name": "bar", "type": "int"}]}' }}
|
16
|
-
|
22
|
+
let (:test_event_hash) { { "foo" => "hello", "bar" => 10 } }
|
23
|
+
let (:test_event) {LogStash::Event.new(test_event_hash)}
|
17
24
|
|
18
|
-
|
19
|
-
|
25
|
+
subject do
|
26
|
+
allow_any_instance_of(LogStash::Codecs::Avro).to \
|
20
27
|
receive(:open_and_read).and_return(avro_config['schema_uri'])
|
21
|
-
|
22
|
-
|
28
|
+
next LogStash::Codecs::Avro.new(avro_config)
|
29
|
+
end
|
23
30
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
31
|
+
context "#decode" do
|
32
|
+
it "should return an LogStash::Event from raw and base64 encoded avro data" do
|
33
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
34
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
35
|
+
buffer = StringIO.new
|
36
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
37
|
+
dw.write(test_event.to_hash, encoder)
|
38
|
+
|
39
|
+
subject.decode(Base64.strict_encode64(buffer.string)) do |event|
|
40
|
+
insist {event.is_a? LogStash::Event}
|
41
|
+
insist {event.get("foo")} == test_event.get("foo")
|
42
|
+
insist {event.get("bar")} == test_event.get("bar")
|
43
|
+
expect(event.get('[event][original]')).to eq(Base64.strict_encode64(buffer.string)) if ecs_compatibility != :disabled
|
44
|
+
end
|
45
|
+
subject.decode(buffer.string) do |event|
|
46
|
+
insist {event.is_a? LogStash::Event}
|
47
|
+
insist {event.get("foo")} == test_event.get("foo")
|
48
|
+
insist {event.get("bar")} == test_event.get("bar")
|
49
|
+
expect(event.get('[event][original]')).to eq(buffer.string) if ecs_compatibility != :disabled
|
50
|
+
end
|
36
51
|
end
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
insist {event.get("bar")} == test_event.get("bar")
|
52
|
+
|
53
|
+
it "should throw exception if decoding fails" do
|
54
|
+
expect {subject.decode("not avro") {|_| }}.to raise_error NoMethodError
|
41
55
|
end
|
42
56
|
end
|
43
57
|
|
44
|
-
|
45
|
-
|
58
|
+
context "with binary encoding" do
|
59
|
+
let (:avro_config) { super().merge('encoding' => 'binary') }
|
60
|
+
|
61
|
+
it "should return an LogStash::Event from raw and base64 encoded avro data" do
|
62
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
63
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
64
|
+
buffer = StringIO.new
|
65
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
66
|
+
dw.write(test_event.to_hash, encoder)
|
67
|
+
|
68
|
+
subject.decode(buffer.string) do |event|
|
69
|
+
expect(event).to be_a_kind_of(LogStash::Event)
|
70
|
+
expect(event.get("foo")).to eq(test_event.get("foo"))
|
71
|
+
expect(event.get("bar")).to eq(test_event.get("bar"))
|
72
|
+
expect(event.get('[event][original]')).to eq(buffer.string) if ecs_compatibility != :disabled
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
it "should raise an error if base64 encoded data is provided" do
|
77
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
78
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
79
|
+
buffer = StringIO.new
|
80
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
81
|
+
dw.write(test_event.to_hash, encoder)
|
82
|
+
|
83
|
+
expect {subject.decode(Base64.strict_encode64(buffer.string))}.to raise_error
|
84
|
+
end
|
46
85
|
end
|
47
|
-
end
|
48
86
|
|
49
|
-
|
50
|
-
|
87
|
+
context "#decode with tag_on_failure" do
|
88
|
+
let (:avro_config) {{ 'schema_uri' => '
|
89
|
+
{"type": "record", "name": "Test",
|
90
|
+
"fields": [{"name": "foo", "type": ["null", "string"]},
|
91
|
+
{"name": "bar", "type": "int"}]}',
|
92
|
+
'tag_on_failure' => true}}
|
51
93
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
94
|
+
it "should tag event on failure" do
|
95
|
+
subject.decode("not avro") do |event|
|
96
|
+
insist {event.is_a? LogStash::Event}
|
97
|
+
insist {event.get("tags")} == ["_avroparsefailure"]
|
98
|
+
end
|
56
99
|
end
|
57
100
|
end
|
58
|
-
end
|
59
101
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
102
|
+
context "#decode with target" do
|
103
|
+
let(:avro_target) { "avro_target" }
|
104
|
+
let (:avro_config) {{ 'schema_uri' => '
|
105
|
+
{"type": "record", "name": "Test",
|
106
|
+
"fields": [{"name": "foo", "type": ["null", "string"]},
|
107
|
+
{"name": "bar", "type": "int"}]}',
|
108
|
+
'target' => avro_target}}
|
109
|
+
|
110
|
+
it "should return an LogStash::Event with content in target" do
|
64
111
|
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
112
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
113
|
+
buffer = StringIO.new
|
114
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
115
|
+
dw.write(test_event.to_hash, encoder)
|
116
|
+
|
117
|
+
subject.decode(buffer.string) do |event|
|
118
|
+
insist {event.get("[#{avro_target}][foo]")} == test_event.get("foo")
|
119
|
+
insist {event.get("[#{avro_target}][bar]")} == test_event.get("bar")
|
120
|
+
end
|
74
121
|
end
|
75
|
-
subject.encode(test_event)
|
76
|
-
insist {got_event}
|
77
122
|
end
|
78
123
|
|
79
|
-
context "
|
124
|
+
context "#encode" do
|
125
|
+
it "should return avro data from a LogStash::Event" do
|
126
|
+
got_event = false
|
127
|
+
subject.on_event do |event, data|
|
128
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
129
|
+
datum = StringIO.new(Base64.strict_decode64(data))
|
130
|
+
decoder = Avro::IO::BinaryDecoder.new(datum)
|
131
|
+
datum_reader = Avro::IO::DatumReader.new(schema)
|
132
|
+
record = datum_reader.read(decoder)
|
133
|
+
|
134
|
+
insist {record["foo"]} == test_event.get("foo")
|
135
|
+
insist {record["bar"]} == test_event.get("bar")
|
136
|
+
insist {event.is_a? LogStash::Event}
|
137
|
+
got_event = true
|
138
|
+
end
|
139
|
+
subject.encode(test_event)
|
140
|
+
insist {got_event}
|
141
|
+
end
|
142
|
+
|
143
|
+
context "with binary encoding" do
|
144
|
+
let (:avro_config) { super().merge('encoding' => 'binary') }
|
145
|
+
|
146
|
+
it "should return avro data from a LogStash::Event not base64 encoded" do
|
147
|
+
got_event = false
|
148
|
+
subject.on_event do |event, data|
|
149
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
150
|
+
datum = StringIO.new(data)
|
151
|
+
decoder = Avro::IO::BinaryDecoder.new(datum)
|
152
|
+
datum_reader = Avro::IO::DatumReader.new(schema)
|
153
|
+
record = datum_reader.read(decoder)
|
80
154
|
|
81
|
-
|
155
|
+
expect(event).to be_a_kind_of(LogStash::Event)
|
156
|
+
expect(event.get("foo")).to eq(test_event.get("foo"))
|
157
|
+
expect(event.get("bar")).to eq(test_event.get("bar"))
|
158
|
+
got_event = true
|
159
|
+
end
|
160
|
+
subject.encode(test_event)
|
161
|
+
expect(got_event).to be true
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
context "binary data" do
|
166
|
+
|
167
|
+
let (:avro_config) {{ 'schema_uri' => '{"namespace": "com.systems.test.data",
|
82
168
|
"type": "record",
|
83
169
|
"name": "TestRecord",
|
84
170
|
"fields": [
|
@@ -87,29 +173,31 @@ describe LogStash::Codecs::Avro do
|
|
87
173
|
{"name": "latitude", "type": ["double", "null"]}
|
88
174
|
]
|
89
175
|
}' }}
|
90
|
-
|
176
|
+
let (:test_event) {LogStash::Event.new({ "name" => "foo", "longitude" => 21.01234.to_f, "latitude" => 111.0123.to_f })}
|
91
177
|
|
92
|
-
|
93
|
-
|
178
|
+
subject do
|
179
|
+
allow_any_instance_of(LogStash::Codecs::Avro).to \
|
94
180
|
receive(:open_and_read).and_return(avro_config['schema_uri'])
|
95
|
-
|
96
|
-
|
181
|
+
next LogStash::Codecs::Avro.new(avro_config)
|
182
|
+
end
|
97
183
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
184
|
+
it "should correctly encode binary data" do
|
185
|
+
schema = Avro::Schema.parse(avro_config['schema_uri'])
|
186
|
+
dw = Avro::IO::DatumWriter.new(schema)
|
187
|
+
buffer = StringIO.new
|
188
|
+
encoder = Avro::IO::BinaryEncoder.new(buffer)
|
189
|
+
dw.write(test_event.to_hash, encoder)
|
104
190
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
191
|
+
subject.decode(Base64.strict_encode64(buffer.string)) do |event|
|
192
|
+
insist {event.is_a? LogStash::Event}
|
193
|
+
insist {event.get("name")} == test_event.get("name")
|
194
|
+
insist {event.get("longitude")} == test_event.get("longitude")
|
195
|
+
insist {event.get("latitude")} == test_event.get("latitude")
|
196
|
+
end
|
110
197
|
end
|
111
198
|
end
|
112
199
|
end
|
200
|
+
|
113
201
|
end
|
114
202
|
end
|
115
203
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-avro
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.4.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -33,17 +33,59 @@ dependencies:
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
35
35
|
requirements:
|
36
|
-
- - "
|
36
|
+
- - "~>"
|
37
37
|
- !ruby/object:Gem::Version
|
38
|
-
version:
|
38
|
+
version: 1.10.2
|
39
39
|
name: avro
|
40
40
|
prerelease: false
|
41
41
|
type: :runtime
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- - "
|
44
|
+
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version:
|
46
|
+
version: 1.10.2
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - "~>"
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: '1.3'
|
53
|
+
name: logstash-mixin-ecs_compatibility_support
|
54
|
+
prerelease: false
|
55
|
+
type: :runtime
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '1.3'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - "~>"
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '1.0'
|
67
|
+
name: logstash-mixin-event_support
|
68
|
+
prerelease: false
|
69
|
+
type: :runtime
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '1.0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
requirement: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - "~>"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '1.0'
|
81
|
+
name: logstash-mixin-validator_support
|
82
|
+
prerelease: false
|
83
|
+
type: :runtime
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '1.0'
|
47
89
|
- !ruby/object:Gem::Dependency
|
48
90
|
requirement: !ruby/object:Gem::Requirement
|
49
91
|
requirements:
|
@@ -112,8 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
112
154
|
- !ruby/object:Gem::Version
|
113
155
|
version: '0'
|
114
156
|
requirements: []
|
115
|
-
|
116
|
-
rubygems_version: 2.6.13
|
157
|
+
rubygems_version: 3.1.6
|
117
158
|
signing_key:
|
118
159
|
specification_version: 4
|
119
160
|
summary: Reads serialized Avro records as Logstash events
|