logstash-codec-protobuf 0.1.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -3
- data/Gemfile +1 -0
- data/lib/logstash/codecs/protobuf.rb +102 -77
- data/logstash-codec-protobuf.gemspec +2 -3
- data/spec/codecs/protobuf_spec.rb +0 -29
- data/spec/helpers/unicorn.pb.rb +0 -1
- metadata +13 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 43ef1232f802b173bf7fee33f894c31145e5032e
|
4
|
+
data.tar.gz: f0d955aeee52c1bbb6f72abe016603c6eac8994e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fb9a3a3b646935acd74d986e3defd67a79ccb14bad91bbd9abe4240bf938ef001a709a8ec05efa243410c25d08e3577a1c3a53e1614bccda55d0a6f07837e719
|
7
|
+
data.tar.gz: ad7d0690b3169ab22623db7b5e8dfd5a17d2cb1e082d54c39da86fef8239dd3576a1d1ed5894dcd27f3fe332f1538d8a71a960cb41a7ee85ab7f5f05c32dc4ef
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'logstash/codecs/base'
|
3
3
|
require 'logstash/util/charset'
|
4
|
-
require 'protocol_buffers'
|
4
|
+
require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers
|
5
5
|
|
6
6
|
# This codec converts protobuf encoded messages into logstash events and vice versa.
|
7
7
|
#
|
@@ -20,6 +20,7 @@ require 'protocol_buffers'
|
|
20
20
|
# }
|
21
21
|
# }
|
22
22
|
#
|
23
|
+
|
23
24
|
class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
|
24
25
|
config_name 'protobuf'
|
25
26
|
|
@@ -64,124 +65,153 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
|
|
64
65
|
#
|
65
66
|
config :include_path, :validate => :array, :required => true
|
66
67
|
|
68
|
+
|
69
|
+
|
70
|
+
|
71
|
+
|
67
72
|
def register
|
68
|
-
@
|
69
|
-
include_path.each { |path|
|
70
|
-
@
|
71
|
-
|
73
|
+
@pb_metainfo = {}
|
74
|
+
include_path.each { |path| require_pb_path(path) }
|
75
|
+
@obj = create_object_from_name(class_name)
|
76
|
+
@logger.debug("Protobuf files successfully loaded.")
|
77
|
+
|
72
78
|
end
|
73
79
|
|
74
80
|
def decode(data)
|
75
|
-
decoded = @
|
76
|
-
|
81
|
+
decoded = @obj.parse(data.to_s)
|
82
|
+
results = keys2strings(decoded.to_hash)
|
83
|
+
yield LogStash::Event.new(results) if block_given?
|
77
84
|
end # def decode
|
78
85
|
|
86
|
+
def keys2strings(data)
|
87
|
+
if data.is_a?(::Hash)
|
88
|
+
new_hash = Hash.new
|
89
|
+
data.each{|k,v| new_hash[k.to_s] = keys2strings(v)}
|
90
|
+
new_hash
|
91
|
+
else
|
92
|
+
data
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
79
96
|
|
80
97
|
def encode(event)
|
98
|
+
protobytes = generate_protobuf(event)
|
99
|
+
@on_event.call(event, protobytes)
|
100
|
+
end # def encode
|
101
|
+
|
102
|
+
private
|
103
|
+
def generate_protobuf(event)
|
104
|
+
meth = self.method("encoder_strategy_1")
|
105
|
+
data = meth.call(event, @class_name)
|
81
106
|
begin
|
82
|
-
|
83
|
-
|
84
|
-
protobytes = pbo.serialize_to_string
|
85
|
-
@on_event.call(event, protobytes)
|
107
|
+
msg = @obj.new(data)
|
108
|
+
msg.serialize_to_string
|
86
109
|
rescue NoMethodError
|
87
|
-
@logger.
|
88
|
-
rescue => e
|
89
|
-
@logger.warn("Could not encode protobuf: " + e.message)
|
110
|
+
@logger.debug("error 2: NoMethodError. Maybe mismatching protobuf definition. Required fields are: " + event.to_hash.keys.join(", "))
|
90
111
|
end
|
91
|
-
end
|
112
|
+
end
|
92
113
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
114
|
+
def encoder_strategy_1(event, class_name)
|
115
|
+
_encoder_strategy_1(event.to_hash, class_name)
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
def _encoder_strategy_1(datahash, class_name)
|
120
|
+
fields = clean_hash_keys(datahash)
|
121
|
+
fields = flatten_hash_values(fields) # TODO we could merge this and the above method back into one to save one iteration, but how are we going to name it?
|
122
|
+
meta = get_complex_types(class_name) # returns a hash with member names and their protobuf class names
|
123
|
+
meta.map do | (k,typeinfo) |
|
99
124
|
if fields.include?(k)
|
100
|
-
|
101
|
-
proto_obj =
|
125
|
+
original_value = fields[k]
|
126
|
+
proto_obj = create_object_from_name(typeinfo)
|
102
127
|
fields[k] =
|
103
|
-
if
|
104
|
-
|
105
|
-
|
106
|
-
# put back into the list.
|
107
|
-
value.map { |x| prepare_nested_objects(x, class_name) }
|
108
|
-
value
|
128
|
+
if original_value.is_a?(::Array)
|
129
|
+
ecs1_list_helper(original_value, proto_obj, typeinfo)
|
130
|
+
|
109
131
|
else
|
110
|
-
|
132
|
+
recursive_fix = _encoder_strategy_1(original_value, class_name)
|
133
|
+
proto_obj.new(recursive_fix)
|
111
134
|
end # if is array
|
112
135
|
end
|
136
|
+
|
113
137
|
end
|
138
|
+
|
114
139
|
fields
|
115
140
|
end
|
116
141
|
|
142
|
+
def ecs1_list_helper(value, proto_obj, class_name)
|
143
|
+
# make this field an array/list of protobuf objects
|
144
|
+
# value is a list of hashed complex objects, each of which needs to be protobuffed and
|
145
|
+
# put back into the list.
|
146
|
+
next unless value.is_a?(::Array)
|
147
|
+
value.map { |x| _encoder_strategy_1(x, class_name) }
|
148
|
+
value
|
149
|
+
end
|
117
150
|
|
118
|
-
|
119
|
-
|
120
|
-
# Necessary for @timestamp fields and the likes. Otherwise we'd run into errors (no such method) upon creating the protobuf object.
|
121
|
-
# Then convert timestamps and other objects to strings so that they can be passed to the protobuf object constructor method.
|
122
|
-
def prepare_for_encoding(datahash)
|
151
|
+
def flatten_hash_values(datahash)
|
152
|
+
# 2) convert timestamps and other objects to strings
|
123
153
|
next unless datahash.is_a?(::Hash)
|
124
|
-
|
154
|
+
|
155
|
+
::Hash[datahash.map{|(k,v)| [k, (convert_to_string?(v) ? v.to_s : v)] }]
|
125
156
|
end
|
126
157
|
|
127
|
-
def
|
128
|
-
|
129
|
-
|
158
|
+
def clean_hash_keys(datahash)
|
159
|
+
# 1) remove @ signs from keys
|
160
|
+
next unless datahash.is_a?(::Hash)
|
161
|
+
|
162
|
+
::Hash[datahash.map{|(k,v)| [remove_atchar(k.to_s), v] }]
|
163
|
+
end #clean_hash_keys
|
130
164
|
|
131
|
-
def
|
165
|
+
def convert_to_string?(v)
|
132
166
|
!(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
|
133
167
|
end
|
134
168
|
|
135
|
-
|
136
|
-
def
|
169
|
+
|
170
|
+
def remove_atchar(key) # necessary for @timestamp fields and the likes. Protobuf definition doesn't handle @ in field names well.
|
171
|
+
key.dup.gsub(/@/,'')
|
172
|
+
end
|
173
|
+
|
174
|
+
private
|
175
|
+
def create_object_from_name(name)
|
137
176
|
begin
|
138
|
-
|
177
|
+
@logger.debug("Creating instance of " + name)
|
139
178
|
name.split('::').inject(Object) { |n,c| n.const_get c }
|
140
179
|
end
|
141
180
|
end
|
142
181
|
|
182
|
+
def get_complex_types(class_name)
|
183
|
+
@pb_metainfo[class_name]
|
184
|
+
end
|
143
185
|
|
144
|
-
|
145
|
-
|
146
|
-
# This is needed for the encoder section of the codec.
|
147
|
-
# When encoding an event into a pb class which uses other pb classes, we need to create the
|
148
|
-
# objects for those nested classes first, so that we can reference them when encoding the topmost
|
149
|
-
# class. In order to be able to do so, this method reads each protobuf class line by line and
|
150
|
-
# stores the information in the @pb_class_references member.
|
151
|
-
# Params:
|
152
|
-
# +filename+:: the absolute path to the protobuf definition.
|
153
|
-
def load_class_reference_information(filename)
|
186
|
+
def require_with_metadata_analysis(filename)
|
187
|
+
require filename
|
154
188
|
regex_class_name = /\s*class\s*(?<name>.+?)\s+/
|
155
189
|
regex_module_name = /\s*module\s*(?<name>.+?)\s+/
|
156
190
|
regex_pbdefs = /\s*(optional|repeated)(\s*):(?<type>.+),(\s*):(?<name>\w+),(\s*)(?<position>\d+)/
|
191
|
+
# now we also need to find out which class it contains and the protobuf definitions in it.
|
192
|
+
# We'll unfortunately need that later so that we can create nested objects.
|
157
193
|
begin
|
158
194
|
class_name = ""
|
159
195
|
type = ""
|
160
196
|
field_name = ""
|
161
197
|
classname_found = false
|
162
198
|
File.readlines(filename).each do |line|
|
163
|
-
|
164
|
-
if ! (line =~ regex_module_name).nil? && !classname_found
|
165
|
-
# Module name found, so we start to create the class name string which starts with the module.
|
199
|
+
if ! (line =~ regex_module_name).nil? && !classname_found # because it might be declared twice in the file
|
166
200
|
class_name << $1
|
167
201
|
class_name << "::"
|
202
|
+
|
168
203
|
end
|
169
|
-
|
170
|
-
# Check if the current line contains the class name (but only if it hasn't been found yet because it might be declared twice in the file)
|
171
|
-
if ! (line =~ regex_class_name).nil? && !classname_found
|
172
|
-
# class name found. Let's append it to the class name string, which might already contain the module name
|
204
|
+
if ! (line =~ regex_class_name).nil? && !classname_found # because it might be declared twice in the file
|
173
205
|
class_name << $1
|
174
|
-
|
175
|
-
@pb_class_references[class_name] = {}
|
206
|
+
@pb_metainfo[class_name] = {}
|
176
207
|
classname_found = true
|
177
208
|
end
|
178
|
-
|
179
209
|
if ! (line =~ regex_pbdefs).nil?
|
180
210
|
type = $1
|
181
211
|
field_name = $2
|
182
212
|
if type =~ /::/
|
183
|
-
|
184
|
-
|
213
|
+
@pb_metainfo[class_name][field_name] = type.gsub!(/^:/,"")
|
214
|
+
|
185
215
|
end
|
186
216
|
end
|
187
217
|
end
|
@@ -193,21 +223,16 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
|
|
193
223
|
end
|
194
224
|
end
|
195
225
|
|
196
|
-
|
197
|
-
|
198
|
-
# Params:
|
199
|
-
# +dir_or_file+:: the absolute path to the file or directory that need to be loaded
|
200
|
-
def load_protobuf_classfiles(dir_or_file)
|
226
|
+
def require_pb_path(dir_or_file)
|
227
|
+
f = dir_or_file.end_with? ('.rb')
|
201
228
|
begin
|
202
|
-
if
|
203
|
-
|
204
|
-
|
205
|
-
load_class_reference_information dir_or_file
|
229
|
+
if f
|
230
|
+
@logger.debug("Including protobuf file: " + dir_or_file)
|
231
|
+
require_with_metadata_analysis dir_or_file
|
206
232
|
else
|
207
233
|
Dir[ dir_or_file + '/*.rb'].each { |file|
|
208
|
-
|
209
|
-
|
210
|
-
load_class_reference_information file
|
234
|
+
@logger.debug("Including protobuf path: " + dir_or_file + "/" + file)
|
235
|
+
require_with_metadata_analysis file
|
211
236
|
}
|
212
237
|
end
|
213
238
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-protobuf'
|
4
|
-
s.version = '0.
|
4
|
+
s.version = '1.0.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -20,8 +20,7 @@ Gem::Specification.new do |s|
|
|
20
20
|
|
21
21
|
# Gem dependencies
|
22
22
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
23
|
-
s.add_runtime_dependency 'ruby-protocol-buffers' #
|
24
|
-
# s.add_runtime_dependency 'google-protobuf' # https://github.com/google/protobuf/tree/master/ruby
|
23
|
+
s.add_runtime_dependency 'ruby-protocol-buffers' # used by the compiled version of our protobuf definition.
|
25
24
|
s.add_development_dependency 'logstash-devutils'
|
26
25
|
end
|
27
26
|
|
@@ -27,7 +27,6 @@ describe LogStash::Codecs::Protobuf do
|
|
27
27
|
expect(event.get("horn_length") ).to eq(data[:horn_length] )
|
28
28
|
expect(event.get("last_seen") ).to eq(data[:last_seen] )
|
29
29
|
expect(event.get("has_wings") ).to eq(data[:has_wings] )
|
30
|
-
|
31
30
|
end
|
32
31
|
end # it
|
33
32
|
|
@@ -57,8 +56,6 @@ describe LogStash::Codecs::Protobuf do
|
|
57
56
|
plugin_human.decode(hugo.serialize_to_string) do |event|
|
58
57
|
expect(event.get("first_name") ).to eq(data[:first_name] )
|
59
58
|
expect(event.get("middle_names") ).to eq(data[:middle_names] )
|
60
|
-
expect(event.get("middle_names").length ).to eq(data[:middle_names].length )
|
61
|
-
expect(event.get("[middle_names]")[1] ).to eq(data[:middle_names][1] )
|
62
59
|
expect(event.get("last_name") ).to eq(data[:last_name] )
|
63
60
|
expect(event.get("[mother][first_name]") ).to eq(data_m[:first_name] )
|
64
61
|
expect(event.get("[father][first_name]") ).to eq(data_f[:first_name] )
|
@@ -205,30 +202,4 @@ describe LogStash::Codecs::Protobuf do
|
|
205
202
|
|
206
203
|
|
207
204
|
|
208
|
-
context "#encode4" do
|
209
|
-
subject do
|
210
|
-
next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/ColourTestcase.pb.rb'])
|
211
|
-
end
|
212
|
-
|
213
|
-
require 'spec/helpers/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
|
214
|
-
event = LogStash::Event.new("booleantest" => [false, false, true], "least_liked" => ColourProtoTest::Colour::YELLOW, "favourite_colours" => \
|
215
|
-
[ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )
|
216
|
-
|
217
|
-
it "should return protobuf encoded data from a complex event with enums" do
|
218
|
-
|
219
|
-
subject.on_event do |event, data|
|
220
|
-
insist { data.is_a? String }
|
221
|
-
|
222
|
-
colpref = ColourProtoTest.parse(data)
|
223
|
-
|
224
|
-
expect(colpref.booleantest ).to eq(event.get("booleantest") )
|
225
|
-
expect(colpref.least_liked ).to eq(event.get("least_liked") )
|
226
|
-
expect(colpref.favourite_colours ).to eq(event.get("favourite_colours") )
|
227
|
-
|
228
|
-
|
229
|
-
end # subject.on_event
|
230
|
-
subject.encode(event)
|
231
|
-
end # it
|
232
|
-
end # context
|
233
|
-
|
234
205
|
end
|
data/spec/helpers/unicorn.pb.rb
CHANGED
metadata
CHANGED
@@ -1,22 +1,22 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-protobuf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Inga Feick
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-12-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
|
-
- -
|
16
|
+
- - ">="
|
17
17
|
- !ruby/object:Gem::Version
|
18
18
|
version: '1.60'
|
19
|
-
- - <=
|
19
|
+
- - "<="
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '2.99'
|
22
22
|
name: logstash-core-plugin-api
|
@@ -24,16 +24,16 @@ dependencies:
|
|
24
24
|
type: :runtime
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
|
-
- -
|
27
|
+
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '1.60'
|
30
|
-
- - <=
|
30
|
+
- - "<="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
35
35
|
requirements:
|
36
|
-
- -
|
36
|
+
- - ">="
|
37
37
|
- !ruby/object:Gem::Version
|
38
38
|
version: '0'
|
39
39
|
name: ruby-protocol-buffers
|
@@ -41,13 +41,13 @@ dependencies:
|
|
41
41
|
type: :runtime
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- -
|
44
|
+
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '0'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
49
49
|
requirements:
|
50
|
-
- -
|
50
|
+
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: '0'
|
53
53
|
name: logstash-devutils
|
@@ -55,7 +55,7 @@ dependencies:
|
|
55
55
|
type: :development
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
|
-
- -
|
58
|
+
- - ">="
|
59
59
|
- !ruby/object:Gem::Version
|
60
60
|
version: '0'
|
61
61
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
@@ -91,17 +91,17 @@ require_paths:
|
|
91
91
|
- lib
|
92
92
|
required_ruby_version: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
98
98
|
requirements:
|
99
|
-
- -
|
99
|
+
- - ">="
|
100
100
|
- !ruby/object:Gem::Version
|
101
101
|
version: '0'
|
102
102
|
requirements: []
|
103
103
|
rubyforge_project:
|
104
|
-
rubygems_version: 2.4.
|
104
|
+
rubygems_version: 2.4.8
|
105
105
|
signing_key:
|
106
106
|
specification_version: 4
|
107
107
|
summary: This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages
|