logstash-codec-protobuf 0.1.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0571d20de470f854fce5c65a479811846d0c3223
4
- data.tar.gz: 4d0447fadbd0d520ce33b299253712c7beae65d4
3
+ metadata.gz: 43ef1232f802b173bf7fee33f894c31145e5032e
4
+ data.tar.gz: f0d955aeee52c1bbb6f72abe016603c6eac8994e
5
5
  SHA512:
6
- metadata.gz: defc91159982c93264d62fddfdbd4677dac242e2b6a49e310275c5cb7095e2b2be25f88cfd5e40f27381bce100711bab21ec9b96a76da6f5a7c838d481a438ee
7
- data.tar.gz: 0923ffd9b10199f952ca12dad21e040a0b049b10129dd9332a3078a7269f73e9d8e7b163aaa7c8d205802ffe0c91aee861a47ea411c8d418e3cb8732cbf3f356
6
+ metadata.gz: fb9a3a3b646935acd74d986e3defd67a79ccb14bad91bbd9abe4240bf938ef001a709a8ec05efa243410c25d08e3577a1c3a53e1614bccda55d0a6f07837e719
7
+ data.tar.gz: ad7d0690b3169ab22623db7b5e8dfd5a17d2cb1e082d54c39da86fef8239dd3576a1d1ed5894dcd27f3fe332f1538d8a71a960cb41a7ee85ab7f5f05c32dc4ef
data/CHANGELOG.md CHANGED
@@ -1,5 +1,5 @@
1
- ## 0.1.3
2
- - Changes for compatibility with logstash 5: remove mechanism to guarantee string based hash keys.
3
-
1
+ ## 1.0.0
2
+ - Update to v5.0 API
3
+
4
4
  ## 0.1.2
5
5
  - First version of this plugin
data/Gemfile CHANGED
@@ -1,2 +1,3 @@
1
1
  source 'https://rubygems.org'
2
+ gem 'logstash-docgen', :path => "/Users/suyog/ws/elastic/logstash/tools/logstash-docgen"
2
3
  gemspec
@@ -1,7 +1,7 @@
1
1
  # encoding: utf-8
2
2
  require 'logstash/codecs/base'
3
3
  require 'logstash/util/charset'
4
- require 'protocol_buffers'
4
+ require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers
5
5
 
6
6
  # This codec converts protobuf encoded messages into logstash events and vice versa.
7
7
  #
@@ -20,6 +20,7 @@ require 'protocol_buffers'
20
20
  # }
21
21
  # }
22
22
  #
23
+
23
24
  class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
24
25
  config_name 'protobuf'
25
26
 
@@ -64,124 +65,153 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
64
65
  #
65
66
  config :include_path, :validate => :array, :required => true
66
67
 
68
+
69
+
70
+
71
+
67
72
  def register
68
- @pb_class_references = {}
69
- include_path.each { |path| load_protobuf_classfiles(path) }
70
- @protobuf_class = create_protobuf_object(@class_name)
71
-
73
+ @pb_metainfo = {}
74
+ include_path.each { |path| require_pb_path(path) }
75
+ @obj = create_object_from_name(class_name)
76
+ @logger.debug("Protobuf files successfully loaded.")
77
+
72
78
  end
73
79
 
74
80
  def decode(data)
75
- decoded = @protobuf_class.parse(data.to_s)
76
- yield LogStash::Event.new(decoded.to_hash) if block_given?
81
+ decoded = @obj.parse(data.to_s)
82
+ results = keys2strings(decoded.to_hash)
83
+ yield LogStash::Event.new(results) if block_given?
77
84
  end # def decode
78
85
 
86
+ def keys2strings(data)
87
+ if data.is_a?(::Hash)
88
+ new_hash = Hash.new
89
+ data.each{|k,v| new_hash[k.to_s] = keys2strings(v)}
90
+ new_hash
91
+ else
92
+ data
93
+ end
94
+ end
95
+
79
96
 
80
97
  def encode(event)
98
+ protobytes = generate_protobuf(event)
99
+ @on_event.call(event, protobytes)
100
+ end # def encode
101
+
102
+ private
103
+ def generate_protobuf(event)
104
+ meth = self.method("encoder_strategy_1")
105
+ data = meth.call(event, @class_name)
81
106
  begin
82
- data = prepare_nested_objects(event.to_hash, @class_name)
83
- pbo = @protobuf_class.new(data)
84
- protobytes = pbo.serialize_to_string
85
- @on_event.call(event, protobytes)
107
+ msg = @obj.new(data)
108
+ msg.serialize_to_string
86
109
  rescue NoMethodError
87
- @logger.warn("Error 2: NoMethodError. Maybe mismatching protobuf definition? Make sure that your protobuf definition has at least these fields: " + event.to_hash.keys.join(", "))
88
- rescue => e
89
- @logger.warn("Could not encode protobuf: " + e.message)
110
+ @logger.debug("error 2: NoMethodError. Maybe mismatching protobuf definition. Required fields are: " + event.to_hash.keys.join(", "))
90
111
  end
91
- end # def encode
112
+ end
92
113
 
93
- # Creates instances of nested protobuf references recursively. TODO improve documentation
94
- private
95
- def prepare_nested_objects(fields, class_name)
96
- fields = prepare_for_encoding(fields)
97
- referenced_classes = @pb_class_references[class_name] # returns a hash with member names and their protobuf class names
98
- referenced_classes.map do | (k,class_name) |
114
+ def encoder_strategy_1(event, class_name)
115
+ _encoder_strategy_1(event.to_hash, class_name)
116
+
117
+ end
118
+
119
+ def _encoder_strategy_1(datahash, class_name)
120
+ fields = clean_hash_keys(datahash)
121
+ fields = flatten_hash_values(fields) # TODO we could merge this and the above method back into one to save one iteration, but how are we going to name it?
122
+ meta = get_complex_types(class_name) # returns a hash with member names and their protobuf class names
123
+ meta.map do | (k,typeinfo) |
99
124
  if fields.include?(k)
100
- value = fields[k]
101
- proto_obj = create_protobuf_object(class_name)
125
+ original_value = fields[k]
126
+ proto_obj = create_object_from_name(typeinfo)
102
127
  fields[k] =
103
- if value.is_a?(::Array)
104
- # make this field an array/list of protobuf objects
105
- # value is a list of hashed complex objects, each of which needs to be protobuffed and
106
- # put back into the list.
107
- value.map { |x| prepare_nested_objects(x, class_name) }
108
- value
128
+ if original_value.is_a?(::Array)
129
+ ecs1_list_helper(original_value, proto_obj, typeinfo)
130
+
109
131
  else
110
- proto_obj.new( prepare_nested_objects(value, class_name) )
132
+ recursive_fix = _encoder_strategy_1(original_value, class_name)
133
+ proto_obj.new(recursive_fix)
111
134
  end # if is array
112
135
  end
136
+
113
137
  end
138
+
114
139
  fields
115
140
  end
116
141
 
142
+ def ecs1_list_helper(value, proto_obj, class_name)
143
+ # make this field an array/list of protobuf objects
144
+ # value is a list of hashed complex objects, each of which needs to be protobuffed and
145
+ # put back into the list.
146
+ next unless value.is_a?(::Array)
147
+ value.map { |x| _encoder_strategy_1(x, class_name) }
148
+ value
149
+ end
117
150
 
118
-
119
- # Removes @ characters from the member names of the event.
120
- # Necessary for @timestamp fields and the likes. Otherwise we'd run into errors (no such method) upon creating the protobuf object.
121
- # Then convert timestamps and other objects to strings so that they can be passed to the protobuf object constructor method.
122
- def prepare_for_encoding(datahash)
151
+ def flatten_hash_values(datahash)
152
+ # 2) convert timestamps and other objects to strings
123
153
  next unless datahash.is_a?(::Hash)
124
- ::Hash[datahash.map{|(k,v)| [k.to_s.gsub(/@/,'').to_sym, convert_value(v)] }]
154
+
155
+ ::Hash[datahash.map{|(k,v)| [k, (convert_to_string?(v) ? v.to_s : v)] }]
125
156
  end
126
157
 
127
- def convert_value(v)
128
- (convertable_to_string?(v) ? v.to_s : v)
129
- end
158
+ def clean_hash_keys(datahash)
159
+ # 1) remove @ signs from keys
160
+ next unless datahash.is_a?(::Hash)
161
+
162
+ ::Hash[datahash.map{|(k,v)| [remove_atchar(k.to_s), v] }]
163
+ end #clean_hash_keys
130
164
 
131
- def convertable_to_string?(v)
165
+ def convert_to_string?(v)
132
166
  !(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
133
167
  end
134
168
 
135
- # Creates an instance of a protobuf class name. This instance will be used later to call the decode and encode methods on.
136
- def create_protobuf_object(name)
169
+
170
+ def remove_atchar(key) # necessary for @timestamp fields and the likes. Protobuf definition doesn't handle @ in field names well.
171
+ key.dup.gsub(/@/,'')
172
+ end
173
+
174
+ private
175
+ def create_object_from_name(name)
137
176
  begin
138
-
177
+ @logger.debug("Creating instance of " + name)
139
178
  name.split('::').inject(Object) { |n,c| n.const_get c }
140
179
  end
141
180
  end
142
181
 
182
+ def get_complex_types(class_name)
183
+ @pb_metainfo[class_name]
184
+ end
143
185
 
144
-
145
- # Analyses a protobuf definition on which other protobuf classes it uses.
146
- # This is needed for the encoder section of the codec.
147
- # When encoding an event into a pb class which uses other pb classes, we need to create the
148
- # objects for those nested classes first, so that we can reference them when encoding the topmost
149
- # class. In order to be able to do so, this method reads each protobuf class line by line and
150
- # stores the information in the @pb_class_references member.
151
- # Params:
152
- # +filename+:: the absolute path to the protobuf definition.
153
- def load_class_reference_information(filename)
186
+ def require_with_metadata_analysis(filename)
187
+ require filename
154
188
  regex_class_name = /\s*class\s*(?<name>.+?)\s+/
155
189
  regex_module_name = /\s*module\s*(?<name>.+?)\s+/
156
190
  regex_pbdefs = /\s*(optional|repeated)(\s*):(?<type>.+),(\s*):(?<name>\w+),(\s*)(?<position>\d+)/
191
+ # now we also need to find out which class it contains and the protobuf definitions in it.
192
+ # We'll unfortunately need that later so that we can create nested objects.
157
193
  begin
158
194
  class_name = ""
159
195
  type = ""
160
196
  field_name = ""
161
197
  classname_found = false
162
198
  File.readlines(filename).each do |line|
163
- # Check if the current line contains the module name (but only if the class name hasn't been found yet because it might be declared twice in the file)
164
- if ! (line =~ regex_module_name).nil? && !classname_found
165
- # Module name found, so we start to create the class name string which starts with the module.
199
+ if ! (line =~ regex_module_name).nil? && !classname_found # because it might be declared twice in the file
166
200
  class_name << $1
167
201
  class_name << "::"
202
+
168
203
  end
169
-
170
- # Check if the current line contains the class name (but only if it hasn't been found yet because it might be declared twice in the file)
171
- if ! (line =~ regex_class_name).nil? && !classname_found
172
- # class name found. Let's append it to the class name string, which might already contain the module name
204
+ if ! (line =~ regex_class_name).nil? && !classname_found # because it might be declared twice in the file
173
205
  class_name << $1
174
- # initialize the hash for the field specific information that we will collect in the next step
175
- @pb_class_references[class_name] = {}
206
+ @pb_metainfo[class_name] = {}
176
207
  classname_found = true
177
208
  end
178
-
179
209
  if ! (line =~ regex_pbdefs).nil?
180
210
  type = $1
181
211
  field_name = $2
182
212
  if type =~ /::/
183
- # the line contains a field declaration which references another class. We need to store the name of that class.
184
- @pb_class_references[class_name][field_name] = type.gsub!(/^:/,"")
213
+ @pb_metainfo[class_name][field_name] = type.gsub!(/^:/,"")
214
+
185
215
  end
186
216
  end
187
217
  end
@@ -193,21 +223,16 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
193
223
  end
194
224
  end
195
225
 
196
- # This method calls 'require' for the protobuf class files listed in the 'include_path' section of the config.
197
- # When given a directory instead of a file, it will require all files in the directory.
198
- # Params:
199
- # +dir_or_file+:: the absolute path to the file or directory that need to be loaded
200
- def load_protobuf_classfiles(dir_or_file)
226
+ def require_pb_path(dir_or_file)
227
+ f = dir_or_file.end_with? ('.rb')
201
228
  begin
202
- if dir_or_file.end_with? ('.rb')
203
-
204
- require dir_or_file
205
- load_class_reference_information dir_or_file
229
+ if f
230
+ @logger.debug("Including protobuf file: " + dir_or_file)
231
+ require_with_metadata_analysis dir_or_file
206
232
  else
207
233
  Dir[ dir_or_file + '/*.rb'].each { |file|
208
-
209
- require file
210
- load_class_reference_information file
234
+ @logger.debug("Including protobuf path: " + dir_or_file + "/" + file)
235
+ require_with_metadata_analysis file
211
236
  }
212
237
  end
213
238
  end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-protobuf'
4
- s.version = '0.1.3'
4
+ s.version = '1.0.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages"
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,8 +20,7 @@ Gem::Specification.new do |s|
20
20
 
21
21
  # Gem dependencies
22
22
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
23
- s.add_runtime_dependency 'ruby-protocol-buffers' # https://github.com/codekitchen/ruby-protocol-buffers
24
- # s.add_runtime_dependency 'google-protobuf' # https://github.com/google/protobuf/tree/master/ruby
23
+ s.add_runtime_dependency 'ruby-protocol-buffers' # used by the compiled version of our protobuf definition.
25
24
  s.add_development_dependency 'logstash-devutils'
26
25
  end
27
26
 
@@ -27,7 +27,6 @@ describe LogStash::Codecs::Protobuf do
27
27
  expect(event.get("horn_length") ).to eq(data[:horn_length] )
28
28
  expect(event.get("last_seen") ).to eq(data[:last_seen] )
29
29
  expect(event.get("has_wings") ).to eq(data[:has_wings] )
30
-
31
30
  end
32
31
  end # it
33
32
 
@@ -57,8 +56,6 @@ describe LogStash::Codecs::Protobuf do
57
56
  plugin_human.decode(hugo.serialize_to_string) do |event|
58
57
  expect(event.get("first_name") ).to eq(data[:first_name] )
59
58
  expect(event.get("middle_names") ).to eq(data[:middle_names] )
60
- expect(event.get("middle_names").length ).to eq(data[:middle_names].length )
61
- expect(event.get("[middle_names]")[1] ).to eq(data[:middle_names][1] )
62
59
  expect(event.get("last_name") ).to eq(data[:last_name] )
63
60
  expect(event.get("[mother][first_name]") ).to eq(data_m[:first_name] )
64
61
  expect(event.get("[father][first_name]") ).to eq(data_f[:first_name] )
@@ -205,30 +202,4 @@ describe LogStash::Codecs::Protobuf do
205
202
 
206
203
 
207
204
 
208
- context "#encode4" do
209
- subject do
210
- next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/ColourTestcase.pb.rb'])
211
- end
212
-
213
- require 'spec/helpers/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
214
- event = LogStash::Event.new("booleantest" => [false, false, true], "least_liked" => ColourProtoTest::Colour::YELLOW, "favourite_colours" => \
215
- [ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )
216
-
217
- it "should return protobuf encoded data from a complex event with enums" do
218
-
219
- subject.on_event do |event, data|
220
- insist { data.is_a? String }
221
-
222
- colpref = ColourProtoTest.parse(data)
223
-
224
- expect(colpref.booleantest ).to eq(event.get("booleantest") )
225
- expect(colpref.least_liked ).to eq(event.get("least_liked") )
226
- expect(colpref.favourite_colours ).to eq(event.get("favourite_colours") )
227
-
228
-
229
- end # subject.on_event
230
- subject.encode(event)
231
- end # it
232
- end # context
233
-
234
205
  end
@@ -14,7 +14,6 @@ module Animal
14
14
  optional :int32, :horn_length, 2
15
15
  optional :int32, :last_seen, 3
16
16
  optional :bool, :has_wings, 4
17
- optional :float, :height, 5
18
17
  end
19
18
 
20
19
  end
metadata CHANGED
@@ -1,22 +1,22 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-protobuf
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Inga Feick
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-28 00:00:00.000000000 Z
11
+ date: 2016-12-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - '>='
16
+ - - ">="
17
17
  - !ruby/object:Gem::Version
18
18
  version: '1.60'
19
- - - <=
19
+ - - "<="
20
20
  - !ruby/object:Gem::Version
21
21
  version: '2.99'
22
22
  name: logstash-core-plugin-api
@@ -24,16 +24,16 @@ dependencies:
24
24
  type: :runtime
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
- - - '>='
27
+ - - ">="
28
28
  - !ruby/object:Gem::Version
29
29
  version: '1.60'
30
- - - <=
30
+ - - "<="
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2.99'
33
33
  - !ruby/object:Gem::Dependency
34
34
  requirement: !ruby/object:Gem::Requirement
35
35
  requirements:
36
- - - '>='
36
+ - - ">="
37
37
  - !ruby/object:Gem::Version
38
38
  version: '0'
39
39
  name: ruby-protocol-buffers
@@ -41,13 +41,13 @@ dependencies:
41
41
  type: :runtime
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - '>='
44
+ - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0'
47
47
  - !ruby/object:Gem::Dependency
48
48
  requirement: !ruby/object:Gem::Requirement
49
49
  requirements:
50
- - - '>='
50
+ - - ">="
51
51
  - !ruby/object:Gem::Version
52
52
  version: '0'
53
53
  name: logstash-devutils
@@ -55,7 +55,7 @@ dependencies:
55
55
  type: :development
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - '>='
58
+ - - ">="
59
59
  - !ruby/object:Gem::Version
60
60
  version: '0'
61
61
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -91,17 +91,17 @@ require_paths:
91
91
  - lib
92
92
  required_ruby_version: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - '>='
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  required_rubygems_version: !ruby/object:Gem::Requirement
98
98
  requirements:
99
- - - '>='
99
+ - - ">="
100
100
  - !ruby/object:Gem::Version
101
101
  version: '0'
102
102
  requirements: []
103
103
  rubyforge_project:
104
- rubygems_version: 2.4.5
104
+ rubygems_version: 2.4.8
105
105
  signing_key:
106
106
  specification_version: 4
107
107
  summary: This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages