logstash-codec-protobuf 1.2.2 → 1.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4a059736022035d5e326d6cb27ddec13cd54da37dcb1d2f37a8076cb0179e37
-  data.tar.gz: 3402775465ce05bbbbc87a7367acc482cd7d2468c84e27ad07ef0ac4383e8a97
+  metadata.gz: afe37c419ffac934a07356c24954ab9534d6438363a8cbc3a4a9a9f7b4214cd0
+  data.tar.gz: 775f81231ce4181e43900251e5fea9906027efc5cbff2d65cd82d122b28fd977
 SHA512:
-  metadata.gz: 4b1dea1ed7701b7e6e26e8226991bc718fab06302071f60b7c8b8b0b4b9a2b45d1d0809be6fb7a76a30af0c291181a5a77cb8a51cdf2a4c5e9ddc2596c0afc1e
-  data.tar.gz: 778c28f5ed7cfa1ed5bd54f175f8efec4dc74e0d93e1d47dc3ded5671a2e61c6baf4e2f9b6bf9e87e896324f3b2a3e22c3a3692195b62fcb23a2bc37815f26aa
+  metadata.gz: 02cff8a64b2bf3324d456b173b8b52761fb6ba8501f87e0011cc02328d75322552c794206fafa574919dc04e7bfe30fbb0f59e804a00f4550c57730913ff80fa
+  data.tar.gz: a431bd7300fa3dae4c16c82fd1dab79eefe8b1ea3552d5b9f9c8af175862867d029bd2ac398c1af445ec67e59cf12d010e61d5f75a3c159cfa64e294f7c1e178

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 1.2.5
+  - Encoder bugfix: avoid pipeline crash if encoding failed.
+
+## 1.2.4
+  - Encoder bugfix: avoid pipeline crash if encoding failed.
+
+## 1.2.3
+  - Add oneof information to @metadata (protobuf version 3 only).
+
 ## 1.2.2
   - Add type conversion feature to encoder
 

data/Gemfile

@@ -8,4 +8,4 @@ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "
 if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
-end
+end

data/LICENSE

@@ -179,7 +179,7 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (c) 2012-2018 Elasticsearch http://www.elastic.co
+   Copyright 2020 Elastic and contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -200,4 +200,3 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-

data/docs/index.asciidoc

@@ -207,8 +207,32 @@ Stop entire pipeline when encountering a non decodable message.
 
 Convert data types to match the protobuf definition (if possible).
 The protobuf encoder library is very strict with regards to data types. Example: an event has an integer field but the protobuf definition expects a float. This would lead to an exception and the event would be lost.
+
 This feature tries to convert the datatypes to the expectations of the protobuf definitions, without modifying the data whatsoever. Examples of conversions it might attempt:
-"true" string => true boolean
-17 int => 17.0 float
-12345 number => "12345" string
+
+  `"true" :: string => true :: boolean`
+
+  `17 :: int => 17.0 :: float`
+
+  `12345 :: number => "12345" :: string`
+
+Available only for protobuf version 3.
+
+[id="plugins-{type}s-{plugin}-pb3_set_oneof_metainfo"]
+===== `pb3_set_oneof_metainfo`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is false
+
+Add meta information to `[@metadata][pb_oneof]` about which classes were chosen for [oneof](https://developers.google.com/protocol-buffers/docs/proto3#oneof) fields. A new field of name `[@metadata][pb_oneof][FOO]` will be added, where `FOO` is the name of the `oneof` field.
+
+Example values: for the protobuf definition
+[source,ruby]
+    oneof :horse_type do
+      optional :unicorn, :message, 2, "UnicornType"
+      optional :pegasus, :message, 3, "PegasusType"
+    end
+
+the field `[@metadata][pb_oneof][horse_type]` will be set to either `pegasus` or `unicorn`.
+Available only for protobuf version 3.
 

data/lib/logstash/codecs/protobuf.rb

@@ -134,12 +134,22 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   # Recommendation: use the translate plugin to restore previous behaviour when upgrading.
   config :protobuf_version, :validate => [2,3], :default => 2, :required => true
 
-  # To tolerate faulty messages that cannot be decoded, set this to false. Otherwise the pipeline will stop upon encountering a non decipherable message.
+  # To tolerate faulty messages that cannot be en/decoded, set this to false. Otherwise the pipeline will stop upon encountering a non decipherable message.
   config :stop_on_error, :validate => :boolean, :default => false, :required => false
 
   # Instruct the encoder to attempt converting data types to match the protobuf definitions. Available only for protobuf version 3.
   config :pb3_encoder_autoconvert_types, :validate => :boolean, :default => true, :required => false
 
+  # Add meta information to `[@metadata][pb_oneof]` about which classes were chosen for [oneof](https://developers.google.com/protocol-buffers/docs/proto3#oneof) fields.
+  # Example values: for the protobuf definition
+  # ```    oneof :horse_type do
+  #          optional :unicorn, :message, 2, "FantasyUnicorn"
+  #          optional :pegasus, :message, 3, "FantasyPegasus"
+  #        end
+  # ```
+  # the field `[@metadata][pb_oneof][horse_type]` will be set to either `pegasus` or `unicorn`.
+  # Available only for protobuf version 3.
+  config :pb3_set_oneof_metainfo, :validate => :boolean, :default => false, :required => false
 
 
   attr_reader :execution_context
@@ -155,7 +165,6 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     @metainfo_pb2_enumlist = []
     @pb3_typeconversion_tag = "_protobuf_type_converted"
 
-
     if @include_path.length > 0 and not class_file.strip.empty?
       raise LogStash::ConfigurationError, "Cannot use `include_path` and `class_file` at the same time"
     end
@@ -203,16 +212,23 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   def decode(data)
     if @protobuf_version == 3
       decoded = @pb_builder.decode(data.to_s)
+      if @pb3_set_oneof_metainfo
+        meta = pb3_get_oneof_metainfo(decoded, @class_name)
+      end
       h = pb3_deep_to_hash(decoded)
     else
       decoded = @pb_builder.parse(data.to_s)
       h = decoded.to_hash
     end
-    yield LogStash::Event.new(h) if block_given?
-  rescue => e
-    @logger.warn("Couldn't decode protobuf: #{e.inspect}.")
+    e = LogStash::Event.new(h)
+    if @protobuf_version == 3 and @pb3_set_oneof_metainfo
+      e.set("[@metadata][pb_oneof]", meta)
+    end
+    yield e if block_given?
+  rescue => ex
+    @logger.warn("Couldn't decode protobuf: #{ex.inspect}.")
     if stop_on_error
-      raise e
+      raise ex
     else # keep original message so that the user can debug it.
       yield LogStash::Event.new("message" => data, "tags" => ["_protobufdecodefailure"])
     end
@@ -225,7 +241,9 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     else
       protobytes = pb2_encode(event)
     end
-    @on_event.call(event, protobytes)
+    unless protobytes.nil? or protobytes.empty?
+      @on_event.call(event, protobytes)
+    end
   end # def encode
 
 
@@ -277,10 +295,13 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     k = event.to_hash.keys.join(", ")
     @logger.warn("Protobuf encoding error 1: Argument error (#{e.inspect}). Reason: probably mismatching protobuf definition. \
       Required fields in the protobuf definition are: #{k} and fields must not begin with @ sign. The event has been discarded.")
+    nil
   rescue TypeError => e
     pb3_handle_type_errors(event, e, is_recursive_call, datahash)
+    nil
   rescue => e
     @logger.warn("Protobuf encoding error 3: #{e.inspect}. Event discarded. Input data: #{datahash}. The event has been discarded. Backtrace: #{e.backtrace}")
+    nil
   end
 
 
@@ -297,9 +318,6 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
           @logger.warn(msg)
           mismatches = pb3_get_type_mismatches(datahash, "", @class_name)
 
-          msg = "Protobuf encoding info 2.2: Type mismatches found: #{mismatches}." # TODO remove
-          @logger.warn(msg)
-
           event = pb3_convert_mismatched_types(event, mismatches)
           # Add a (temporary) tag to handle the recursion stop
           pb3_add_tag(event, @pb3_typeconversion_tag )
@@ -314,11 +332,20 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
       else
         @logger.warn("Protobuf encoding error 2.4.2: (#{e.inspect}). The event has been discarded.")
       end
+      if @stop_on_error
+        raise e
+      end
+      nil
     rescue => ex
       @logger.warn("Protobuf encoding error 2.5: (#{e.inspect}). The event has been discarded. Auto-typecasting was on: #{@pb3_encoder_autoconvert_types}")
+      if @stop_on_error
+        raise ex
+      end
+      nil
     end
   end # pb3_handle_type_errors
 
+
   def pb3_get_type_mismatches(data, key_prefix, pb_class)
     mismatches = []
     data.to_hash.each do |key, value|
@@ -354,17 +381,15 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     else
       case value
       when ::Hash, Google::Protobuf::MessageExts
-
         is_mismatch = false
         descriptor = Google::Protobuf::DescriptorPool.generated_pool.lookup(pb_class).lookup(key)
-        if descriptor.subtype != nil
+        if !descriptor.subtype.nil?
           class_of_nested_object = pb3_get_descriptorpool_name(descriptor.subtype.msgclass)
           new_prefix = "#{key}."
           recursive_mismatches = pb3_get_type_mismatches(value, new_prefix, class_of_nested_object)
           mismatches.concat(recursive_mismatches)
         end
       when ::Array
-
         expected_type = pb3_get_expected_type(key, pb_class)
         is_mismatch = (expected_type != Google::Protobuf::RepeatedField)
         child_type = Google::Protobuf::DescriptorPool.generated_pool.lookup(pb_class).lookup(key).type
@@ -510,6 +535,29 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     datahash
   end
 
+  def pb3_get_oneof_metainfo(pb_object, pb_class_name)
+    meta = {}
+    pb_class = Google::Protobuf::DescriptorPool.generated_pool.lookup(pb_class_name).msgclass
+
+    pb_class.descriptor.each_oneof { |field|
+      field.each { | group_option |
+        if !pb_object.send(group_option.name).nil?
+            meta[field.name] = group_option.name
+        end
+      }
+    }
+
+    pb_class.descriptor.select{ |field| field.type == :message }.each { | field |
+      # recurse over nested protobuf classes
+      pb_sub_object = pb_object.send(field.name)
+      if !pb_sub_object.nil? and !field.subtype.nil?
+          pb_sub_class = pb3_get_descriptorpool_name(field.subtype.msgclass)
+          meta[field.name] = pb3_get_oneof_metainfo(pb_sub_object, pb_sub_class)
+      end
+    }
+
+    meta
+  end
 
 
   def pb2_encode(event)
@@ -525,7 +573,6 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   end
 
 
-
   def pb2_prepare_for_encoding(datahash, class_name)
     if datahash.is_a?(::Hash)
       # Preparation: the data cannot be encoded until certain criteria are met:

data/logstash-codec-protobuf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '1.2.2'
+  s.version         = '1.2.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads protobuf messages and converts to Logstash Events"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -23,5 +23,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'google-protobuf', '3.5.0.pre'
   s.add_runtime_dependency 'ruby-protocol-buffers' # for protobuf 2
   s.add_development_dependency 'logstash-devutils'
-end
 
+end

data/spec/codecs/pb2_spec.rb

@@ -2,8 +2,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/protobuf"
 require "logstash/event"
-require "insist"
-
 
 require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers, for protobuf2
 
@@ -117,7 +115,7 @@ describe LogStash::Codecs::Protobuf do
 
     it "should return protobuf encoded data from a simple event" do
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
         unicorn = Animal::UnicornEvent.parse(data)
 
         expect(unicorn.colour ).to eq(event.get("colour") )
@@ -148,7 +146,7 @@ describe LogStash::Codecs::Protobuf do
     it "should return protobuf encoded data from a complex event" do
 
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
         jimmy = Animal::Human.parse(data)
 
         expect(jimmy.first_name ).to eq(event.get("first_name") )
@@ -190,7 +188,7 @@ describe LogStash::Codecs::Protobuf do
     it "should return protobuf encoded data from a complex event with enums" do
 
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
 
         colpref = ColourProtoTest.parse(data)
 

data/spec/codecs/pb3_decode_spec.rb

@@ -2,7 +2,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/protobuf"
 require "logstash/event"
-require "insist"
 
 require 'google/protobuf' # for protobuf3
 
@@ -74,7 +73,7 @@ describe LogStash::Codecs::Protobuf do
     end
   end # context
 
-  context "#test1_pb3" do
+  context "#pb3decoder_test1" do
 
 
     #### Test case 1: Decode simple protobuf ####################################################################################################################
@@ -102,7 +101,7 @@ describe LogStash::Codecs::Protobuf do
     end # it
   end # context
 
-  context "#test2_pb3" do
+  context "#pb3decoder_test2" do
 
     #### Test case 2: decode nested protobuf ####################################################################################################################
     let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)  }
@@ -125,7 +124,7 @@ describe LogStash::Codecs::Protobuf do
 
   end # context
 
-  context "#test3_pb3" do
+  context "#pb3decoder_test3" do
 
     #### Test case 3: decode ProbeResult ####################################################################################################################
     let(:plugin_3) { LogStash::Codecs::Protobuf.new("class_name" => "ProbeResult", "include_path" => [pb_include_path + '/pb3/ProbeResult_pb.rb'], "protobuf_version" => 3)  }
@@ -157,7 +156,7 @@ describe LogStash::Codecs::Protobuf do
     end # it
   end # context
 
-  context "#test4_pb3" do
+  context "#pb3decoder_test4" do
 
     #### Test case 4: decode PBDNSMessage ####################################################################################################################
     let(:plugin_4) { LogStash::Codecs::Protobuf.new("class_name" => "PBDNSMessage", "include_path" => [pb_include_path + '/pb3/dnsmessage_pb.rb'], "protobuf_version" => 3)  }
@@ -238,7 +237,7 @@ describe LogStash::Codecs::Protobuf do
 
   end # context
 
-  context "#test5_pb3" do
+  context "#pb3decoder_test5" do
 
     #### Test case 5: decode test case for github issue 17 ####################################################################################################################
     let(:plugin_5) { LogStash::Codecs::Protobuf.new("class_name" => "com.foo.bar.IntegerTestMessage", "include_path" => [pb_include_path + '/pb3/integertest_pb.rb'], "protobuf_version" => 3)  }
@@ -259,7 +258,7 @@ describe LogStash::Codecs::Protobuf do
 
   end # context
 
-  context "#test6_pb3" do
+  context "#pb3decoder_test6" do
 
 
     let(:execution_context) { double("execution_context")}
@@ -307,7 +306,7 @@ describe LogStash::Codecs::Protobuf do
 
 
 
-  context "#test7_pb3" do
+  context "#pb3decoder_test7" do
 
     #### Test case 6: decode test case for github issue 17 ####################################################################################################################
     let(:plugin_7) { LogStash::Codecs::Protobuf.new("class_name" => "RepeatedEvents", "include_path" => [pb_include_path + '/pb3/events_pb.rb'], "protobuf_version" => 3)  }
@@ -334,7 +333,113 @@ describe LogStash::Codecs::Protobuf do
     end # it
 
 
-  end # context test7_pb3
+  end # context pb3decoder_test7
 
 
+  context "#pb3decoder_test8a" do
+
+    ########################################################################################################################
+    let(:plugin_8a) { LogStash::Codecs::Protobuf.new("class_name" => "FantasyHorse", "class_file" => 'pb3/FantasyHorse_pb.rb',
+      "protobuf_root_directory" => pb_include_path, "protobuf_version" => 3, "pb3_set_oneof_metainfo" => true)  }
+    before do
+        plugin_8a.register
+    end
+
+    it "should add meta information on oneof fields" do
+      pegasus_data = {:wings_length => 100}
+      horsey = FantasyPegasus.new(pegasus_data)
+
+      braid_data = {:braid_thickness => 10, :braiding_style => "french"}
+      tail_data = {:tail_length => 80, :braided => BraidedHorseTail.new(braid_data) }
+      tail = FantasyHorseTail.new(tail_data)
+
+      data = {:name=>"Reinhold", :pegasus => horsey, :tail => tail}
+      pb_obj = FantasyHorse.new(data)
+      bin = FantasyHorse.encode(pb_obj)
+      plugin_8a.decode(bin) do |event|
+
+        expect(event.get("name") ).to eq(data[:name])
+        expect(event.get("pegasus")["wings_length"] ).to eq(pegasus_data[:wings_length])
+        expect(event.get("tail")['tail_length'] ).to eq(tail_data[:tail_length])
+        expect(event.get("tail")['braided']['braiding_style'] ).to eq(braid_data[:braiding_style])
+        expect(event.get("@metadata")["pb_oneof"]["horse_type"] ).to eq("pegasus")
+        expect(event.get("@metadata")["pb_oneof"]["tail"]["hair_type"] ).to eq("braided")
+
+      end
+    end # it
+
+
+  end # context pb3decoder_test8a
+
+
+
+
+  context "#pb3decoder_test8b" do
+
+    ########################################################################################################################
+    let(:plugin_8b) { LogStash::Codecs::Protobuf.new("class_name" => "FantasyHorse", "class_file" => 'pb3/FantasyHorse_pb.rb',
+      "protobuf_root_directory" => pb_include_path, "protobuf_version" => 3, "pb3_set_oneof_metainfo" => false)  }
+    before do
+        plugin_8b.register
+    end
+
+    it "should not add meta information on oneof fields" do
+      pegasus_data = {:wings_length => 100}
+      horsey = FantasyPegasus.new(pegasus_data)
+
+      braid_data = {:braid_thickness => 10, :braiding_style => "french"}
+      tail_data = {:tail_length => 80, :braided => BraidedHorseTail.new(braid_data) }
+      tail = FantasyHorseTail.new(tail_data)
+
+      data = {:name=>"Winfried", :pegasus => horsey, :tail => tail}
+      pb_obj = FantasyHorse.new(data)
+      bin = FantasyHorse.encode(pb_obj)
+      plugin_8b.decode(bin) do |event|
+        expect(event.get("name") ).to eq(data[:name])
+        expect(event.get("pegasus")["wings_length"] ).to eq(pegasus_data[:wings_length])
+        expect(event.get("tail")['tail_length'] ).to eq(tail_data[:tail_length])
+        expect(event.get("tail")['braided']['braiding_style'] ).to eq(braid_data[:braiding_style])
+        expect(event.get("@metadata")["pb_oneof"]).to be_nil
+
+      end
+    end # it
+
+
+  end # context pb3decoder_test8b
+
+
+  context "#pb3decoder_test8c" do # same test as 8a just with different one_of options selected
+
+    ########################################################################################################################
+    let(:plugin_8c) { LogStash::Codecs::Protobuf.new("class_name" => "FantasyHorse", "class_file" => 'pb3/FantasyHorse_pb.rb',
+      "protobuf_root_directory" => pb_include_path, "protobuf_version" => 3, "pb3_set_oneof_metainfo" => true)  }
+    before do
+        plugin_8c.register
+    end
+
+    it "should add meta information on oneof fields" do
+      unicorn_data = {:horn_length => 30}
+      horsey = FantasyUnicorn.new(unicorn_data)
+
+      natural_data = {:wavyness => "B"}
+      tail_data = {:tail_length => 80, :natural => NaturalHorseTail.new(natural_data) }
+      tail = FantasyHorseTail.new(tail_data)
+
+      data = {:name=>"Hubert", :unicorn => horsey, :tail => tail}
+      pb_obj = FantasyHorse.new(data)
+      bin = FantasyHorse.encode(pb_obj)
+      plugin_8c.decode(bin) do |event|
+        expect(event.get("name") ).to eq(data[:name])
+        expect(event.get("unicorn")["horn_length"] ).to eq(unicorn_data[:horn_length])
+        expect(event.get("tail")['tail_length'] ).to eq(tail_data[:tail_length])
+        expect(event.get("tail")['natural']['wavyness'] ).to eq(natural_data[:wavyness])
+        expect(event.get("@metadata")["pb_oneof"]["horse_type"] ).to eq("unicorn")
+        expect(event.get("@metadata")["pb_oneof"]["tail"]["hair_type"] ).to eq("natural")
+
+      end
+    end # it
+
+
+  end # context pb3decoder_test8c
+
 end # describe

data/spec/codecs/pb3_encode_spec.rb

@@ -1,12 +1,7 @@
-
-
-
-
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/protobuf"
 require "logstash/event"
-require "insist"
 
 require 'google/protobuf' # for protobuf3
 
@@ -30,7 +25,7 @@ describe LogStash::Codecs::Protobuf do
     it "should return protobuf encoded data for testcase 1" do
 
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
 
         pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
         decoded_data = pb_builder.decode(data)
@@ -64,7 +59,7 @@ describe LogStash::Codecs::Protobuf do
     it "should return protobuf encoded data for testcase 2" do
 
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
 
         pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
         decoded_data = pb_builder.decode(data)
@@ -109,7 +104,7 @@ describe LogStash::Codecs::Protobuf do
     it "should return protobuf encoded data for testcase 3" do
 
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
 
         pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("something.rum_akamai.ProtoAkamaiRum").msgclass
         decoded_data = pb_builder.decode(data)
@@ -160,7 +155,7 @@ describe LogStash::Codecs::Protobuf do
     it "should fix datatypes to match the protobuf definition" do
 
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
 
         pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("something.rum_akamai.ProtoAkamai2Rum").msgclass
         decoded_data = pb_builder.decode(data)
@@ -199,7 +194,7 @@ context "encodePB3-e" do
     it "should ignore empty fields" do
 
       subject.on_event do |event, data|
-        insist { data.is_a? String }
+        expect(data).to be_a(String)
 
         pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("something.rum_akamai.ProtoAkamai3Rum").msgclass
         decoded_data = pb_builder.decode(data)
@@ -216,5 +211,33 @@ context "encodePB3-e" do
 
 
 
+context "encodePB3-f" do
+
+    #### Test case 5: handle additional fields (discard event without crashing pipeline) ####################################################################################################################
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "something.rum_akamai.ProtoAkamai3Rum",
+        "pb3_encoder_autoconvert_types" => false,
+        "include_path" => [pb_include_path + '/pb3/rum3_pb.rb' ], "protobuf_version" => 3)
+    end
+
+    event = LogStash::Event.new(
+      "domain" => nil, "bot" => "This field does not exist in the protobuf definition",
+      "header" => {"sender_id" => "23"},
+      "geo"=>{"organisation"=>"Jio", "rg"=>"DL", "netspeed"=>nil, "city"=>nil, "cc"=>"IN", "ovr"=>false, "postalcode"=>"110012", "isp"=>"Jio"}
+    )
+
+    it "should not return data" do
+
+      subject.on_event do |event, data|
+        expect("the on_event method should not be called").to eq("so this code should never be reached")
+      end
+      subject.encode(event)
+    end # it
+
+  end # context #encodePB3-f
+
+
+
 
-end # describe
+end # describe

data/spec/helpers/pb3/FantasyHorse_pb.rb

@@ -0,0 +1,44 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+
+begin; require 'google/protobuf'; rescue LoadError; end
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "FantasyHorse" do
+    optional :name, :string, 1
+    oneof :horse_type do
+      optional :unicorn, :message, 2, "FantasyUnicorn"
+      optional :pegasus, :message, 3, "FantasyPegasus"
+    end
+    optional :tail, :message, 4, "FantasyHorseTail"
+  end
+  add_message "FantasyUnicorn" do
+    optional :horn_length, :int32, 1
+    optional :horn_colour, :string, 2
+  end
+  add_message "FantasyPegasus" do
+    optional :wings_length, :int32, 1
+    optional :wings_width, :int32, 2
+    optional :wings_feathers, :string, 3
+  end
+  add_message "FantasyHorseTail" do
+    optional :tail_length, :int32, 1
+    oneof :hair_type do
+      optional :braided, :message, 2, "BraidedHorseTail"
+      optional :natural, :message, 3, "NaturalHorseTail"
+    end
+  end
+  add_message "BraidedHorseTail" do
+    optional :braiding_style, :string, 1
+    optional :braid_thickness, :int32, 2
+  end
+  add_message "NaturalHorseTail" do
+    optional :wavyness, :string, 1
+  end
+end
+
+FantasyHorse = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyHorse").msgclass
+FantasyUnicorn = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyUnicorn").msgclass
+FantasyPegasus = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyPegasus").msgclass
+FantasyHorseTail = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyHorseTail").msgclass
+BraidedHorseTail = Google::Protobuf::DescriptorPool.generated_pool.lookup("BraidedHorseTail").msgclass
+NaturalHorseTail = Google::Protobuf::DescriptorPool.generated_pool.lookup("NaturalHorseTail").msgclass

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-protobuf
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.2.5
 platform: ruby
 authors:
 - Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-11 00:00:00.000000000 Z
+date: 2021-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -102,8 +102,8 @@ files:
 - spec/helpers/pb2/human.pb.rb
 - spec/helpers/pb2/unicorn.pb.rb
 - spec/helpers/pb2/unicorn_event.pb.rb
+- spec/helpers/pb3/FantasyHorse_pb.rb
 - spec/helpers/pb3/ProbeResult_pb.rb
-- spec/helpers/pb3/ReservationEntry_pb.rb
 - spec/helpers/pb3/dnsmessage_pb.rb
 - spec/helpers/pb3/events.proto3
 - spec/helpers/pb3/events_pb.rb
@@ -141,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: Reads protobuf messages and converts to Logstash Events
@@ -158,8 +158,8 @@ test_files:
 - spec/helpers/pb2/human.pb.rb
 - spec/helpers/pb2/unicorn.pb.rb
 - spec/helpers/pb2/unicorn_event.pb.rb
+- spec/helpers/pb3/FantasyHorse_pb.rb
 - spec/helpers/pb3/ProbeResult_pb.rb
-- spec/helpers/pb3/ReservationEntry_pb.rb
 - spec/helpers/pb3/dnsmessage_pb.rb
 - spec/helpers/pb3/events.proto3
 - spec/helpers/pb3/events_pb.rb

data/spec/helpers/pb3/ReservationEntry_pb.rb

@@ -1,64 +0,0 @@
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_message "bk.sk.pk.reservations.proto.types.v0.ReservationEntry" do
-    optional :id, :string, 1
-    optional :internalAccountId, :string, 2
-    optional :state, :enum, 3, "bk.sk.pk.reservations.proto.types.v0.ReservationState"
-    optional :instructedAmount, :message, 4, "bk.sk.pk.reservations.proto.types.v0.InstructedAmount"
-    optional :lifetime, :message, 5, "bk.sk.pk.reservations.proto.types.v0.Lifetime"
-    optional :requestor, :message, 6, "bk.sk.pk.reservations.proto.types.v0.Requestor"
-    optional :description, :message, 7, "bk.sk.pk.reservations.proto.types.v0.Description"
-    optional :forceMarker, :bool, 8
-    optional :creationTimestamp, :string, 9
-  end
-  add_message "bk.sk.pk.reservations.proto.types.v0.Requestor" do
-    optional :productCode, :string, 1
-    optional :systemCode, :string, 2
-    optional :init, :string, 3
-  end
-  add_message "bk.sk.pk.reservations.proto.types.v0.Lifetime" do
-    optional :startDateTime, :string, 1
-    optional :endDateTime, :string, 2
-  end
-  add_message "bk.sk.pk.reservations.proto.types.v0.InstructedAmount" do
-    optional :amount, :message, 1, "bk.sk.pk.reservations.proto.types.v0.DecimalNumber"
-    optional :currency, :string, 2
-  end
-  add_message "bk.sk.pk.reservations.proto.types.v0.DecimalNumber" do
-    optional :unscaledValue, :int64, 1
-    optional :scale, :int32, 2
-  end
-  add_message "bk.sk.pk.reservations.proto.types.v0.Description" do
-    optional :text1, :string, 1
-    optional :text2, :string, 2
-  end
-  add_enum "bk.sk.pk.reservations.proto.types.v0.ReservationState" do
-    value :RESERVED, 0
-    value :CANCELED, 1
-    value :CONSUMED, 2
-    value :EXPIRED, 3
-  end
-end
-
-module Bk
-  module Sk
-    module Pk
-      module Reservations
-        module Proto
-          module Types
-            module V0
-              ReservationEntry = Google::Protobuf::DescriptorPool.generated_pool.lookup("bk.sk.pk.reservations.proto.types.v0.ReservationEntry").msgclass
-              Requestor = Google::Protobuf::DescriptorPool.generated_pool.lookup("bk.sk.pk.reservations.proto.types.v0.Requestor").msgclass
-              Lifetime = Google::Protobuf::DescriptorPool.generated_pool.lookup("bk.sk.pk.reservations.proto.types.v0.Lifetime").msgclass
-              InstructedAmount = Google::Protobuf::DescriptorPool.generated_pool.lookup("bk.sk.pk.reservations.proto.types.v0.InstructedAmount").msgclass
-              DecimalNumber = Google::Protobuf::DescriptorPool.generated_pool.lookup("bk.sk.pk.reservations.proto.types.v0.DecimalNumber").msgclass
-              Description = Google::Protobuf::DescriptorPool.generated_pool.lookup("bk.sk.pk.reservations.proto.types.v0.Description").msgclass
-              ReservationState = Google::Protobuf::DescriptorPool.generated_pool.lookup("bk.sk.pk.reservations.proto.types.v0.ReservationState").enummodule
-            end
-          end
-        end
-      end
-    end
-  end
-end