logstash-codec-protobuf 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +15 -2
- data/docs/index.asciidoc +8 -0
- data/lib/logstash/codecs/protobuf.rb +154 -58
- data/logstash-codec-protobuf.gemspec +1 -2
- data/spec/codecs/protobuf3_spec.rb +143 -48
- data/spec/codecs/protobuf_spec.rb +69 -41
- data/spec/helpers/pb2/event.pb.rb +19 -0
- data/spec/helpers/pb2/event.proto +12 -0
- data/spec/helpers/pb2/header/header.pb.rb +16 -0
- data/spec/helpers/pb2/header/header.proto +8 -0
- data/spec/helpers/pb3/header/header.proto3 +7 -0
- data/spec/helpers/pb3/header/header_pb.rb +12 -0
- data/spec/helpers/pb3/messageA.proto3 +12 -0
- data/spec/helpers/pb3/messageA_pb.rb +16 -0
- data/spec/helpers/pb3/messageB.proto3 +12 -0
- data/spec/helpers/pb3/messageB_pb.rb +16 -0
- metadata +23 -3
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-protobuf'
|
4
|
-
s.version = '1.
|
4
|
+
s.version = '1.2.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads protobuf messages and converts to Logstash Events"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -20,7 +20,6 @@ Gem::Specification.new do |s|
|
|
20
20
|
|
21
21
|
# Gem dependencies
|
22
22
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
23
|
-
# s.add_runtime_dependency 'google-protobuf', '3.1'
|
24
23
|
s.add_runtime_dependency 'google-protobuf', '3.5.0.pre'
|
25
24
|
s.add_runtime_dependency 'ruby-protocol-buffers' # for protobuf 2
|
26
25
|
s.add_development_dependency 'logstash-devutils'
|
@@ -4,30 +4,91 @@ require "logstash/codecs/protobuf"
|
|
4
4
|
require "logstash/event"
|
5
5
|
require "insist"
|
6
6
|
|
7
|
-
|
8
7
|
require 'google/protobuf' # for protobuf3
|
9
8
|
|
9
|
+
# absolute path to the protobuf helpers directory
|
10
|
+
pb_include_path = File.expand_path(".") + "/spec/helpers"
|
11
|
+
|
12
|
+
require pb_include_path + '/pb3/unicorn_pb.rb'
|
13
|
+
unicorn_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
|
10
14
|
|
11
15
|
describe LogStash::Codecs::Protobuf do
|
12
16
|
|
13
|
-
|
17
|
+
context ".reloadable?" do
|
18
|
+
subject do
|
19
|
+
next LogStash::Codecs::Protobuf.new(
|
20
|
+
"class_name" => "Unicorn",
|
21
|
+
"include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'],
|
22
|
+
"protobuf_version" => 3
|
23
|
+
)
|
24
|
+
end
|
25
|
+
|
26
|
+
it "returns false" do
|
27
|
+
expect(subject.reloadable?).to be_falsey
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
context "config" do
|
32
|
+
context "using class_file and include_path" do
|
33
|
+
let(:plugin) {
|
34
|
+
LogStash::Codecs::Protobuf.new(
|
35
|
+
"class_name" => "Unicorn",
|
36
|
+
"include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'],
|
37
|
+
"class_file" => pb_include_path + '/pb3/unicorn_pb.rb',
|
38
|
+
"protobuf_version" => 3
|
39
|
+
)
|
40
|
+
}
|
41
|
+
|
42
|
+
it "should fail to register the plugin with ConfigurationError" do
|
43
|
+
expect {plugin.register}.to raise_error(LogStash::ConfigurationError, /`include_path` and `class_file`/)
|
44
|
+
end # it
|
45
|
+
end
|
46
|
+
|
47
|
+
context "not using class_file or include_path" do
|
48
|
+
let(:plugin) {
|
49
|
+
LogStash::Codecs::Protobuf.new("class_name" => "Unicorn")
|
50
|
+
}
|
51
|
+
|
52
|
+
it "should fail to register the plugin with ConfigurationError" do
|
53
|
+
expect {plugin.register}.to raise_error(LogStash::ConfigurationError, /`include_path` or `class_file`/)
|
54
|
+
end # it
|
55
|
+
end
|
56
|
+
|
57
|
+
RSpec::Expectations.configuration.on_potential_false_positives = :nothing
|
58
|
+
|
59
|
+
context "re-registering the plugin with a valid configuration" do
|
60
|
+
let(:plugin) { LogStash::Codecs::Protobuf.new(
|
61
|
+
"class_name" => "A.MessageA",
|
62
|
+
"class_file" => [ pb_include_path + '/pb3/messageA_pb.rb' ],
|
63
|
+
"protobuf_version" => 3,
|
64
|
+
"protobuf_root_directory" => File.expand_path(File.dirname(__FILE__) + pb_include_path + '/pb3/'))
|
65
|
+
}
|
66
|
+
|
67
|
+
it "should not fail" do
|
68
|
+
expect {
|
69
|
+
# this triggers the `register()` method of the plugin multiple times
|
70
|
+
plugin.register
|
71
|
+
plugin.register
|
72
|
+
}.not_to raise_error(RuntimeError)
|
73
|
+
end # it
|
74
|
+
end
|
75
|
+
end # context
|
14
76
|
|
15
77
|
context "#test1_pb3" do
|
16
78
|
|
17
79
|
|
18
80
|
#### Test case 1: Decode simple protobuf ####################################################################################################################
|
19
|
-
let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new(
|
20
|
-
|
21
|
-
|
22
|
-
end
|
81
|
+
let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new(
|
82
|
+
"class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)
|
83
|
+
}
|
23
84
|
|
24
85
|
it "should return an event from protobuf encoded data" do
|
25
|
-
|
86
|
+
|
26
87
|
unicorn_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
|
27
|
-
data = {:name => 'Pinkie', :age => 18, :is_pegasus => false, :favourite_numbers => [4711,23], :fur_colour => Colour::PINK,
|
88
|
+
data = {:name => 'Pinkie', :age => 18, :is_pegasus => false, :favourite_numbers => [4711,23], :fur_colour => Colour::PINK,
|
28
89
|
:favourite_colours => [Colour::GREEN, Colour::BLUE]
|
29
90
|
}
|
30
|
-
|
91
|
+
|
31
92
|
unicorn_object = unicorn_class.new(data)
|
32
93
|
bin = unicorn_class.encode(unicorn_object)
|
33
94
|
plugin_unicorn.decode(bin) do |event|
|
@@ -39,27 +100,17 @@ describe LogStash::Codecs::Protobuf do
|
|
39
100
|
expect(event.get("is_pegasus") ).to eq(data[:is_pegasus] )
|
40
101
|
end
|
41
102
|
end # it
|
42
|
-
end # context
|
103
|
+
end # context
|
43
104
|
|
44
105
|
context "#test2_pb3" do
|
45
106
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
107
|
#### Test case 2: decode nested protobuf ####################################################################################################################
|
50
108
|
let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3) }
|
51
|
-
before do
|
52
|
-
plugin_unicorn.register
|
53
|
-
end
|
54
109
|
|
55
110
|
it "should return an event from protobuf encoded data with nested classes" do
|
56
|
-
|
57
|
-
|
58
|
-
unicorn_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
|
59
|
-
|
60
111
|
father = unicorn_class.new({:name=> "Sparkle", :age => 50, :fur_colour => 3 })
|
61
|
-
data = {:name => 'Glitter', :fur_colour => Colour::GLITTER, :father => father}
|
62
|
-
|
112
|
+
data = {:name => 'Glitter', :fur_colour => Colour::GLITTER, :father => father}
|
113
|
+
|
63
114
|
unicorn_object = unicorn_class.new(data)
|
64
115
|
bin = unicorn_class.encode(unicorn_object)
|
65
116
|
plugin_unicorn.decode(bin) do |event|
|
@@ -72,23 +123,23 @@ describe LogStash::Codecs::Protobuf do
|
|
72
123
|
end
|
73
124
|
end # it
|
74
125
|
|
75
|
-
end # context
|
126
|
+
end # context
|
76
127
|
|
77
128
|
context "#test3_pb3" do
|
78
129
|
|
79
130
|
#### Test case 3: decode ProbeResult ####################################################################################################################
|
80
131
|
let(:plugin_3) { LogStash::Codecs::Protobuf.new("class_name" => "ProbeResult", "include_path" => [pb_include_path + '/pb3/ProbeResult_pb.rb'], "protobuf_version" => 3) }
|
132
|
+
|
81
133
|
before do
|
82
|
-
plugin_3.register
|
134
|
+
plugin_3.register
|
83
135
|
end
|
84
136
|
|
85
137
|
it "should return an event from protobuf encoded data with nested classes" do
|
86
|
-
|
87
138
|
|
88
139
|
probe_result_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("ProbeResult").msgclass
|
89
140
|
ping_result_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PingIPv4Result").msgclass
|
90
141
|
|
91
|
-
ping_result_data = {:status=> PingIPv4Result::Status::ERROR,
|
142
|
+
ping_result_data = {:status=> PingIPv4Result::Status::ERROR,
|
92
143
|
:latency => 50, :ip => "8.8.8.8", :probe_ip => "127.0.0.1", :geolocation => "New York City" }
|
93
144
|
ping_result_object = ping_result_class.new(ping_result_data)
|
94
145
|
|
@@ -104,29 +155,30 @@ describe LogStash::Codecs::Protobuf do
|
|
104
155
|
expect(event.get("TaskPingIPv4Result")["geolocation"] ).to eq(ping_result_data[:geolocation] )
|
105
156
|
end
|
106
157
|
end # it
|
107
|
-
end # context
|
158
|
+
end # context
|
108
159
|
|
109
160
|
context "#test4_pb3" do
|
110
161
|
|
111
162
|
#### Test case 4: decode PBDNSMessage ####################################################################################################################
|
112
163
|
let(:plugin_4) { LogStash::Codecs::Protobuf.new("class_name" => "PBDNSMessage", "include_path" => [pb_include_path + '/pb3/dnsmessage_pb.rb'], "protobuf_version" => 3) }
|
164
|
+
|
113
165
|
before do
|
114
|
-
plugin_4.register
|
166
|
+
plugin_4.register
|
115
167
|
end
|
116
168
|
|
117
169
|
it "should return an event from protobuf encoded data with nested classes" do
|
118
|
-
|
170
|
+
|
119
171
|
|
120
172
|
pbdns_message_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage").msgclass
|
121
|
-
dns_question_class
|
122
|
-
dns_response_class
|
123
|
-
dns_rr_class
|
173
|
+
dns_question_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSQuestion").msgclass
|
174
|
+
dns_response_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse").msgclass
|
175
|
+
dns_rr_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse.DNSRR").msgclass
|
124
176
|
|
125
177
|
dns_question_data = {:qName => "Foo", :qType => 12345, :qClass => 67890 }
|
126
178
|
dns_question_object = dns_question_class.new(dns_question_data)
|
127
179
|
|
128
|
-
dns_response_data = {:rcode => 12345, :appliedPolicy => "baz", :tags => ["a","b","c"],
|
129
|
-
:queryTimeSec => 123, :queryTimeUsec => 456,
|
180
|
+
dns_response_data = {:rcode => 12345, :appliedPolicy => "baz", :tags => ["a","b","c"],
|
181
|
+
:queryTimeSec => 123, :queryTimeUsec => 456,
|
130
182
|
:appliedPolicyType => PBDNSMessage::PolicyType::NSIP}
|
131
183
|
|
132
184
|
dns_rr_data = [
|
@@ -160,11 +212,11 @@ describe LogStash::Codecs::Protobuf do
|
|
160
212
|
pbdns_message_object = pbdns_message_class.new(pbdns_message_data)
|
161
213
|
bin = pbdns_message_class.encode(pbdns_message_object)
|
162
214
|
plugin_4.decode(bin) do |event|
|
163
|
-
|
164
|
-
['messageId', 'serverIdentity','from','to','inBytes','timeUsec','timeSec','id', 'originalRequestorSubnet', 'requestorId' ,'initialRequestId','deviceIdf'].each { |n|
|
215
|
+
|
216
|
+
['messageId', 'serverIdentity','from','to','inBytes','timeUsec','timeSec','id', 'originalRequestorSubnet', 'requestorId' ,'initialRequestId','deviceIdf'].each { |n|
|
165
217
|
expect(event.get(n)).to eq(pbdns_message_data[n.to_sym] ) }
|
166
218
|
|
167
|
-
# enum test:
|
219
|
+
# enum test:
|
168
220
|
expect(event.get("type") ).to eq("DNSIncomingResponseType" )
|
169
221
|
expect(event.get("socketFamily") ).to eq("INET6" )
|
170
222
|
expect(event.get("socketProtocol") ).to eq("TCP" )
|
@@ -172,11 +224,11 @@ describe LogStash::Codecs::Protobuf do
|
|
172
224
|
expect(event.get("question")["qName"] ).to eq(dns_question_data[:qName] )
|
173
225
|
expect(event.get("question")["qType"] ).to eq(dns_question_data[:qType] )
|
174
226
|
expect(event.get("question")["qClass"] ).to eq(dns_question_data[:qClass] )
|
175
|
-
|
227
|
+
|
176
228
|
['rcode', 'appliedPolicy','tags','queryTimeSec','queryTimeUsec'].each { |n| expect(event.get('response')[n]).to eq(dns_response_data[n.to_sym] ) }
|
177
229
|
expect(event.get("response")['appliedPolicyType'] ).to eq("NSIP" )
|
178
230
|
|
179
|
-
dns_rr_data.each_with_index { | data, index |
|
231
|
+
dns_rr_data.each_with_index { | data, index |
|
180
232
|
found = event.get("response")['rrs'][index]
|
181
233
|
['name', 'type','class','ttl','rdata'].each { |n| expect(found[n]).to eq(data[n.to_sym]) }
|
182
234
|
}
|
@@ -184,14 +236,15 @@ describe LogStash::Codecs::Protobuf do
|
|
184
236
|
end
|
185
237
|
end # it
|
186
238
|
|
187
|
-
end # context
|
239
|
+
end # context
|
188
240
|
|
189
241
|
context "#test5_pb3" do
|
190
242
|
|
191
|
-
#### Test case 5: decode test case for github issue 17 ####################################################################################################################
|
243
|
+
#### Test case 5: decode test case for github issue 17 ####################################################################################################################
|
192
244
|
let(:plugin_5) { LogStash::Codecs::Protobuf.new("class_name" => "com.foo.bar.IntegerTestMessage", "include_path" => [pb_include_path + '/pb3/integertest_pb.rb'], "protobuf_version" => 3) }
|
245
|
+
|
193
246
|
before do
|
194
|
-
|
247
|
+
plugin_5.register
|
195
248
|
end
|
196
249
|
|
197
250
|
it "should return an event from protobuf encoded data with nested classes" do
|
@@ -206,12 +259,53 @@ describe LogStash::Codecs::Protobuf do
|
|
206
259
|
|
207
260
|
end # context
|
208
261
|
|
262
|
+
context "#test6_pb3" do
|
263
|
+
|
264
|
+
let(:execution_context) { double("execution_context")}
|
265
|
+
let(:pipeline_id) {rand(36**8).to_s(36)}
|
266
|
+
|
267
|
+
# Test case 6: decode a message automatically loading the dependencies ##############################################################################
|
268
|
+
let(:plugin) { LogStash::Codecs::Protobuf.new(
|
269
|
+
"class_name" => "A.MessageA",
|
270
|
+
"class_file" => [ 'messageA_pb.rb' ],
|
271
|
+
"protobuf_version" => 3,
|
272
|
+
"protobuf_root_directory" => pb_include_path + '/pb3/')
|
273
|
+
}
|
274
|
+
|
275
|
+
before do
|
276
|
+
allow(plugin).to receive(:execution_context).and_return(execution_context)
|
277
|
+
allow(execution_context).to receive(:pipeline_id).and_return(pipeline_id)
|
278
|
+
|
279
|
+
# this is normally done on the input plugins we "mock" it here to avoid
|
280
|
+
# instantiating a dummy input plugin. See
|
281
|
+
# https://github.com/ph/logstash/blob/37551a89b8137c1dc6fa4fbd992584c363a36065/logstash-core/lib/logstash/inputs/base.rb#L108
|
282
|
+
plugin.execution_context = execution_context
|
283
|
+
end
|
284
|
+
|
285
|
+
it "should return an event from protobuf encoded data" do
|
286
|
+
|
287
|
+
header_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("Header").msgclass
|
288
|
+
header_data = {:name => {'a' => 'b'}}
|
289
|
+
header_object = header_class.new(header_data)
|
290
|
+
|
291
|
+
message_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("A.MessageA").msgclass
|
292
|
+
data = {:name => "Test name", :header => header_object}
|
293
|
+
|
294
|
+
message_object = message_class.new(data)
|
295
|
+
bin = message_class.encode(message_object)
|
296
|
+
|
297
|
+
plugin.decode(bin) do |event|
|
298
|
+
expect(event.get("name") ).to eq(data[:name] )
|
299
|
+
expect(event.get("header")['name'] ).to eq(header_data[:name])
|
300
|
+
end
|
301
|
+
end # it
|
302
|
+
end # context
|
303
|
+
|
209
304
|
|
210
305
|
context "#encodePB3-a" do
|
211
306
|
|
212
307
|
#### Test case 3: encode simple protobuf ####################################################################################################################
|
213
308
|
|
214
|
-
|
215
309
|
require_relative '../helpers/pb3/unicorn_pb.rb'
|
216
310
|
|
217
311
|
subject do
|
@@ -226,7 +320,7 @@ describe LogStash::Codecs::Protobuf do
|
|
226
320
|
insist { data.is_a? String }
|
227
321
|
|
228
322
|
pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
|
229
|
-
decoded_data = pb_builder.decode(data)
|
323
|
+
decoded_data = pb_builder.decode(data)
|
230
324
|
expect(decoded_data.name ).to eq(event.get("name") )
|
231
325
|
expect(decoded_data.age ).to eq(event.get("age") )
|
232
326
|
expect(decoded_data.is_pegasus ).to eq(event.get("is_pegasus") )
|
@@ -234,6 +328,7 @@ describe LogStash::Codecs::Protobuf do
|
|
234
328
|
expect(decoded_data.favourite_numbers ).to eq(event.get("favourite_numbers") )
|
235
329
|
expect(decoded_data.favourite_colours ).to eq([:BLUE,:WHITE] )
|
236
330
|
end # subject.on_event
|
331
|
+
|
237
332
|
subject.encode(event3)
|
238
333
|
end # it
|
239
334
|
|
@@ -250,7 +345,7 @@ describe LogStash::Codecs::Protobuf do
|
|
250
345
|
end
|
251
346
|
|
252
347
|
event4 = LogStash::Event.new("name" => "Horst", "age" => 23, "is_pegasus" => true, "mother" => \
|
253
|
-
{"name" => "Mom", "age" => 47}, "father" => {"name"=> "Daddy", "age"=> 50, "fur_colour" => 3 } # 3 == SILVER
|
348
|
+
{"name" => "Mom", "age" => 47}, "father" => {"name"=> "Daddy", "age"=> 50, "fur_colour" => 3 } # 3 == SILVER
|
254
349
|
)
|
255
350
|
|
256
351
|
it "should return protobuf encoded data for testcase 4" do
|
@@ -259,8 +354,8 @@ describe LogStash::Codecs::Protobuf do
|
|
259
354
|
insist { data.is_a? String }
|
260
355
|
|
261
356
|
pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
|
262
|
-
decoded_data = pb_builder.decode(data)
|
263
|
-
|
357
|
+
decoded_data = pb_builder.decode(data)
|
358
|
+
|
264
359
|
expect(decoded_data.name ).to eq(event.get("name") )
|
265
360
|
expect(decoded_data.age ).to eq(event.get("age") )
|
266
361
|
expect(decoded_data.is_pegasus ).to eq(event.get("is_pegasus") )
|
@@ -270,7 +365,7 @@ describe LogStash::Codecs::Protobuf do
|
|
270
365
|
expect(decoded_data.father.age ).to eq(event.get("father")["age"] )
|
271
366
|
expect(decoded_data.father.fur_colour ).to eq(:SILVER)
|
272
367
|
|
273
|
-
|
368
|
+
|
274
369
|
end # subject4.on_event
|
275
370
|
subject.encode(event4)
|
276
371
|
end # it
|
@@ -7,27 +7,29 @@ require "insist"
|
|
7
7
|
|
8
8
|
require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers, for protobuf2
|
9
9
|
|
10
|
+
require_relative '../helpers/pb2/header/header.pb.rb'
|
11
|
+
require_relative '../helpers/pb2/event.pb.rb'
|
10
12
|
|
11
|
-
describe LogStash::Codecs::Protobuf do
|
12
13
|
|
13
|
-
|
14
|
+
describe LogStash::Codecs::Protobuf do
|
14
15
|
|
16
|
+
pb_include_path = File.expand_path(".") + "/spec/helpers"
|
15
17
|
|
16
18
|
context "#test1" do
|
17
19
|
|
18
|
-
|
20
|
+
|
19
21
|
|
20
22
|
#### Test case 1: Decode simple protobuf bytes for unicorn ####################################################################################################################
|
21
23
|
let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Unicorn", "include_path" => [pb_include_path + '/pb2/unicorn.pb.rb']) }
|
22
24
|
before do
|
23
|
-
plugin_unicorn.register
|
25
|
+
plugin_unicorn.register
|
24
26
|
end
|
25
27
|
|
26
28
|
it "should return an event from protobuf encoded data" do
|
27
|
-
|
29
|
+
|
28
30
|
data = {:colour => 'rainbow', :horn_length => 18, :last_seen => 1420081471, :has_wings => true}
|
29
31
|
unicorn = Animal::Unicorn.new(data)
|
30
|
-
|
32
|
+
|
31
33
|
plugin_unicorn.decode(unicorn.serialize_to_string) do |event|
|
32
34
|
expect(event.get("colour") ).to eq(data[:colour] )
|
33
35
|
expect(event.get("horn_length") ).to eq(data[:horn_length] )
|
@@ -41,29 +43,29 @@ describe LogStash::Codecs::Protobuf do
|
|
41
43
|
#### Test case 2: Decode complex protobuf bytes for human #####################################################################################################################
|
42
44
|
|
43
45
|
context "#test2" do
|
44
|
-
|
45
|
-
|
46
|
+
|
47
|
+
|
46
48
|
let(:plugin_human) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => [pb_include_path + '/pb2/human.pb.rb']) }
|
47
49
|
before do
|
48
|
-
plugin_human.register
|
50
|
+
plugin_human.register
|
49
51
|
end
|
50
52
|
|
51
53
|
it "should return an event from complex nested protobuf encoded data" do
|
52
|
-
|
54
|
+
|
53
55
|
data_gm = {:first_name => 'Elisabeth', :last_name => "Oliveoil", :middle_names => ["Maria","Johanna"], :vegetarian=>true}
|
54
56
|
grandmother = Animal::Human.new(data_gm)
|
55
57
|
data_m = {:first_name => 'Annemarie', :last_name => "Smørebrød", :mother => grandmother}
|
56
58
|
mother = Animal::Human.new(data_m)
|
57
59
|
data_f = {:first_name => 'Karl', :middle_names => ["Theodor-Augustin"], :last_name => "Falkenstein"}
|
58
60
|
father = Animal::Human.new(data_f)
|
59
|
-
data = {:first_name => 'Hugo', :middle_names => ["Heinz", "Peter"], :last_name => "Smørebrød",:father => father, :mother => mother}
|
61
|
+
data = {:first_name => 'Hugo', :middle_names => ["Heinz", "Peter"], :last_name => "Smørebrød",:father => father, :mother => mother}
|
60
62
|
hugo = Animal::Human.new(data)
|
61
|
-
|
63
|
+
|
62
64
|
plugin_human.decode(hugo.serialize_to_string) do |event|
|
63
65
|
expect(event.get("first_name") ).to eq(data[:first_name] )
|
64
66
|
expect(event.get("middle_names") ).to eq(data[:middle_names] )
|
65
67
|
expect(event.get("last_name") ).to eq(data[:last_name] )
|
66
|
-
expect(event.get("[mother][first_name]") ).to eq(data_m[:first_name] )
|
68
|
+
expect(event.get("[mother][first_name]") ).to eq(data_m[:first_name] )
|
67
69
|
expect(event.get("[father][first_name]") ).to eq(data_f[:first_name] )
|
68
70
|
expect(event.get("[mother][last_name]") ).to eq(data_m[:last_name] )
|
69
71
|
expect(event.get("[mother][mother][last_name]") ).to eq(data_gm[:last_name] )
|
@@ -80,19 +82,19 @@ describe LogStash::Codecs::Protobuf do
|
|
80
82
|
#### Test case 3: Decoder test for enums #####################################################################################################################
|
81
83
|
|
82
84
|
context "#test3" do
|
83
|
-
|
84
|
-
|
85
|
+
|
86
|
+
|
85
87
|
let(:plugin_col) { LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => [pb_include_path + '/pb2/ColourTestcase.pb.rb']) }
|
86
88
|
before do
|
87
|
-
plugin_col.register
|
89
|
+
plugin_col.register
|
88
90
|
end
|
89
91
|
|
90
92
|
it "should return an event from protobuf encoded data with enums" do
|
91
|
-
|
93
|
+
|
92
94
|
data = {:least_liked => ColourProtoTest::Colour::YELLOW, :favourite_colours => \
|
93
|
-
[ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE], :booleantest => [true, false, true]}
|
95
|
+
[ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE], :booleantest => [true, false, true]}
|
94
96
|
pb = ColourProtoTest.new(data)
|
95
|
-
|
97
|
+
|
96
98
|
plugin_col.decode(pb.serialize_to_string) do |event|
|
97
99
|
expect(event.get("least_liked") ).to eq(data[:least_liked] )
|
98
100
|
expect(event.get("favourite_colours") ).to eq(data[:favourite_colours] )
|
@@ -104,28 +106,25 @@ describe LogStash::Codecs::Protobuf do
|
|
104
106
|
end # context test3
|
105
107
|
|
106
108
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
109
|
#### Test case 4: Encode simple protobuf bytes for unicorn ####################################################################################################################
|
111
110
|
|
112
111
|
context "#encodePB2-a" do
|
113
112
|
subject do
|
114
|
-
next LogStash::Codecs::Protobuf.new("class_name" => "Animal::UnicornEvent", "include_path" => [pb_include_path + '/pb2/unicorn_event.pb.rb'])
|
113
|
+
next LogStash::Codecs::Protobuf.new("class_name" => "Animal::UnicornEvent", "include_path" => [pb_include_path + '/pb2/unicorn_event.pb.rb'])
|
115
114
|
end
|
116
115
|
|
117
|
-
event = LogStash::Event.new("colour" => "pink", "horn_length" => 12, "last_seen" => 1410081999, "has_wings" => true)
|
116
|
+
event = LogStash::Event.new("colour" => "pink", "horn_length" => 12, "last_seen" => 1410081999, "has_wings" => true)
|
118
117
|
|
119
118
|
it "should return protobuf encoded data from a simple event" do
|
120
119
|
subject.on_event do |event, data|
|
121
120
|
insist { data.is_a? String }
|
122
|
-
unicorn = Animal::UnicornEvent.parse(data)
|
123
|
-
|
121
|
+
unicorn = Animal::UnicornEvent.parse(data)
|
122
|
+
|
124
123
|
expect(unicorn.colour ).to eq(event.get("colour") )
|
125
124
|
expect(unicorn.horn_length ).to eq(event.get("horn_length") )
|
126
125
|
expect(unicorn.last_seen ).to eq(event.get("last_seen") )
|
127
126
|
expect(unicorn.has_wings ).to eq(event.get("has_wings") )
|
128
|
-
|
127
|
+
|
129
128
|
end # subject.on_event
|
130
129
|
subject.encode(event)
|
131
130
|
end # it
|
@@ -135,23 +134,23 @@ describe LogStash::Codecs::Protobuf do
|
|
135
134
|
|
136
135
|
|
137
136
|
#### Test case 5: encode complex protobuf bytes for human #####################################################################################################################
|
138
|
-
|
139
|
-
|
137
|
+
|
138
|
+
|
140
139
|
context "#encodePB2-b" do
|
141
140
|
subject do
|
142
|
-
next LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => [pb_include_path + '/pb2/human.pb.rb'])
|
141
|
+
next LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => [pb_include_path + '/pb2/human.pb.rb'])
|
143
142
|
end
|
144
143
|
|
145
144
|
event = LogStash::Event.new("first_name" => "Jimmy", "middle_names" => ["Bob", "James"], "last_name" => "Doe" \
|
146
145
|
, "mother" => {"first_name" => "Jane", "middle_names" => ["Elizabeth"], "last_name" => "Doe" , "age" => 83, "vegetarian"=> false} \
|
147
|
-
, "father" => {"first_name" => "John", "last_name" => "Doe", "@email" => "character_replacement_test@nothing" })
|
146
|
+
, "father" => {"first_name" => "John", "last_name" => "Doe", "@email" => "character_replacement_test@nothing" })
|
148
147
|
|
149
148
|
it "should return protobuf encoded data from a complex event" do
|
150
149
|
|
151
150
|
subject.on_event do |event, data|
|
152
151
|
insist { data.is_a? String }
|
153
|
-
jimmy = Animal::Human.parse(data)
|
154
|
-
|
152
|
+
jimmy = Animal::Human.parse(data)
|
153
|
+
|
155
154
|
expect(jimmy.first_name ).to eq(event.get("first_name") )
|
156
155
|
expect(jimmy.middle_names ).to eq(event.get("middle_names") )
|
157
156
|
expect(jimmy.last_name ).to eq(event.get("last_name") )
|
@@ -163,7 +162,7 @@ describe LogStash::Codecs::Protobuf do
|
|
163
162
|
expect(jimmy.father.last_name ).to eq(event.get("[father][last_name]") )
|
164
163
|
expect(jimmy.father.email ).to eq(event.get("[father][@email]") ) # recursion test for keys
|
165
164
|
expect(jimmy.mother.last_name ).to eq(event.get("[mother][last_name]") )
|
166
|
-
|
165
|
+
|
167
166
|
end # subject.on_event
|
168
167
|
subject.encode(event)
|
169
168
|
end # it
|
@@ -174,9 +173,9 @@ describe LogStash::Codecs::Protobuf do
|
|
174
173
|
|
175
174
|
|
176
175
|
#### Test case 6: encode enums #########################################################################################################################
|
177
|
-
|
178
176
|
|
179
|
-
|
177
|
+
|
178
|
+
|
180
179
|
context "#encodePB2-c" do
|
181
180
|
subject do
|
182
181
|
next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => [pb_include_path + '/pb2/ColourTestcase.pb.rb'])
|
@@ -184,27 +183,56 @@ describe LogStash::Codecs::Protobuf do
|
|
184
183
|
|
185
184
|
require_relative '../helpers/pb2/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
|
186
185
|
# ^ this import is run from the spec directory, $LOGSTASH_DIR/spec/codecs/
|
187
|
-
|
186
|
+
|
188
187
|
event = LogStash::Event.new("booleantest" => [false, false, true], "least_liked" => ColourProtoTest::Colour::YELLOW, "favourite_colours" => \
|
189
|
-
[ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )
|
188
|
+
[ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )
|
190
189
|
|
191
190
|
it "should return protobuf encoded data from a complex event with enums" do
|
192
191
|
|
193
192
|
subject.on_event do |event, data|
|
194
193
|
insist { data.is_a? String }
|
195
194
|
|
196
|
-
colpref = ColourProtoTest.parse(data)
|
197
|
-
|
195
|
+
colpref = ColourProtoTest.parse(data)
|
196
|
+
|
198
197
|
expect(colpref.booleantest ).to eq(event.get("booleantest") )
|
199
198
|
expect(colpref.least_liked ).to eq(event.get("least_liked") )
|
200
199
|
expect(colpref.favourite_colours ).to eq(event.get("favourite_colours") )
|
201
200
|
|
202
|
-
|
201
|
+
|
203
202
|
end # subject.on_event
|
204
203
|
subject.encode(event)
|
205
204
|
end # it
|
206
205
|
end # context
|
207
206
|
|
207
|
+
#### Test case 7: decode a message automatically loading the dependencies ######################################################################################
|
208
|
+
|
209
|
+
context "#test4" do
|
210
|
+
let(:plugin) { LogStash::Codecs::Protobuf.new(
|
211
|
+
"class_name" => "Logging::Event",
|
212
|
+
"class_file" => [ 'event.pb.rb' ],
|
213
|
+
"protobuf_root_directory" => pb_include_path + '/pb2/')
|
214
|
+
}
|
215
|
+
|
216
|
+
before do
|
217
|
+
plugin.register
|
218
|
+
end
|
219
|
+
|
220
|
+
it "should return an event from protobuf encoded data loading the dependencies" do
|
221
|
+
header = Grpc::Header.new(:protocol => 'https')
|
222
|
+
|
223
|
+
data = {
|
224
|
+
:name => "Test",
|
225
|
+
:header => header,
|
226
|
+
}
|
227
|
+
|
228
|
+
pb = Logging::Event.new(data)
|
229
|
+
|
230
|
+
plugin.decode(pb.serialize_to_string) do |event|
|
231
|
+
expect(event.get("name") ).to eq(data[:name])
|
232
|
+
expect(event.get("header") ).to eq({"protocol"=>"https"})
|
233
|
+
end
|
234
|
+
end # it
|
235
|
+
end # context test4
|
208
236
|
|
209
237
|
|
210
238
|
end # describe
|