logstash-codec-avro-data-file 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7800340e990fa0ec4cdfade51aef94f8ca6ae361
|
4
|
+
data.tar.gz: d114301643eca8e6279f3c0252a3fc6fbc841ffc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1baadb595507445ac476be560738cc4b75239ad7141ae41731ee48e37663b70c2cc36e203ec6f8be5adcbf93c9dc2e27003b963144f2d04993ac92d48b30be14
|
7
|
+
data.tar.gz: 12d9e95dffa44cc8125ae7f99a94eeed15a958719b908ca5624f1328b04500b04eb777efe2bb63337367b63af2036f4a8047172a6157c36a806a85926ed791d9
|
@@ -5,19 +5,21 @@ require 'logstash/codecs/base'
|
|
5
5
|
require 'logstash/namespace'
|
6
6
|
require 'tmpdir'
|
7
7
|
|
8
|
-
#
|
9
|
-
#
|
8
|
+
# == Logstash Codec - Avro Data File
|
9
|
+
#
|
10
|
+
# This plugin is used to process logstash events that represent
|
11
|
+
# Avro data files, like the S3 input can produce.
|
10
12
|
#
|
11
|
-
#
|
13
|
+
# ==== Options
|
12
14
|
#
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
15
|
+
# - ``temporary_directory`` - optional. Specifies a directory to store
|
16
|
+
# temporary files in
|
17
|
+
# - ``decorate_events`` - will add avro schema metadata to the events.
|
16
18
|
#
|
17
|
-
#
|
19
|
+
# ==== Usage
|
18
20
|
#
|
19
|
-
#
|
20
|
-
#
|
21
|
+
# input {
|
22
|
+
# stdin { codec => 'avro-data-file' }
|
21
23
|
# }
|
22
24
|
#
|
23
25
|
class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
|
@@ -27,6 +29,7 @@ class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
|
|
27
29
|
# Set the directory where logstash will store the tmp files before processing them.
|
28
30
|
# default to the current OS temporary directory in linux /tmp/logstash/avro
|
29
31
|
config :temporary_directory, validate: :string, default: File.join(Dir.tmpdir, 'logstash', 'avro')
|
32
|
+
config :decorate_events, validate: :boolean, default: false
|
30
33
|
|
31
34
|
def register
|
32
35
|
require 'fileutils'
|
@@ -43,11 +46,15 @@ class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
|
|
43
46
|
return unless block_given?
|
44
47
|
|
45
48
|
Avro::DataFile.open(tempfile.path, 'r') do |reader|
|
49
|
+
schema = reader.datum_reader.writers_schema
|
46
50
|
reader.each do |avro_message|
|
47
|
-
|
51
|
+
event = LogStash::Event.new(avro_message)
|
52
|
+
decorate_event(event, schema) if decorate_events?
|
53
|
+
yield event
|
48
54
|
end
|
49
55
|
end
|
50
56
|
rescue => e
|
57
|
+
puts e
|
51
58
|
@logger.error('Avro parse error', error: e)
|
52
59
|
ensure
|
53
60
|
reset
|
@@ -76,4 +83,16 @@ class LogStash::Codecs::AvroDataFile < LogStash::Codecs::Base
|
|
76
83
|
end
|
77
84
|
self.tempfile = Tempfile.create('', temporary_directory)
|
78
85
|
end
|
86
|
+
|
87
|
+
def decorate_events?
|
88
|
+
@decorate_events
|
89
|
+
end
|
90
|
+
|
91
|
+
def decorate_event(event, schema)
|
92
|
+
event.set('[@metadata][avro][type]', schema.type)
|
93
|
+
if schema.is_a?(Avro::Schema::NamedSchema)
|
94
|
+
event.set('[@metadata][avro][name]', schema.name)
|
95
|
+
event.set('[@metadata][avro][namespace]', schema.namespace)
|
96
|
+
end
|
97
|
+
end
|
79
98
|
end
|
@@ -52,11 +52,14 @@ describe LogStash::Codecs::AvroDataFile do
|
|
52
52
|
end
|
53
53
|
|
54
54
|
context "valid data" do
|
55
|
+
let(:schema_name) { 'mock_schema' }
|
56
|
+
let(:schema_type) { 'record' }
|
57
|
+
let(:schema_namespace) { 'com.salsify.test' }
|
55
58
|
let(:schema) do
|
56
59
|
{
|
57
|
-
'type' =>
|
58
|
-
'name' =>
|
59
|
-
'namespace' =>
|
60
|
+
'type' => schema_type,
|
61
|
+
'name' => schema_name,
|
62
|
+
'namespace' => schema_namespace,
|
60
63
|
'fields' => [
|
61
64
|
{
|
62
65
|
'name' => 'id',
|
@@ -85,6 +88,21 @@ describe LogStash::Codecs::AvroDataFile do
|
|
85
88
|
let(:expected_events) { avro_messages.map(&LogStash::Event.method(:new)) }
|
86
89
|
|
87
90
|
include_examples "produces the correct output"
|
91
|
+
|
92
|
+
context "decorate events" do
|
93
|
+
let(:config) { { 'decorate_events' => true } }
|
94
|
+
let(:expected_events) do
|
95
|
+
avro_messages.map do |message|
|
96
|
+
LogStash::Event.new(message).tap do |event|
|
97
|
+
event.set('[@metadata][avro][type]', schema_type)
|
98
|
+
event.set('[@metadata][avro][name]', schema_name)
|
99
|
+
event.set('[@metadata][avro][namespace]', schema_namespace)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
include_examples "produces the correct output"
|
105
|
+
end
|
88
106
|
end
|
89
107
|
end
|
90
108
|
end
|