logstash-codec-protobuf 1.2.8-jruby → 1.2.9-jruby

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4282811dfc02dd7f91bf6c7c5f1e959e238bb26adb561ae53032279437af0f8e
-  data.tar.gz: c6941995f4e7be2e4704dc672b38b200cec6cb63d13c8cc94b2ce142f965e1a7
+  metadata.gz: a2ec4c6cbd85449002193bfca0cdb35072fd24cf8c8986f8fea39300ca318e34
+  data.tar.gz: 2b9b98884f6b0534a50d4d917e86983c5aabf471da660e59c6f4e10ea3f5b030
 SHA512:
-  metadata.gz: a1e2fe6f267983100d21eb6933835a039271c0df5fe2a27ac64324b770f9c38b8f8778c435552df20447887b3be84fc560d94139fecf5aee8054284c98debbcd
-  data.tar.gz: 2627426ef1cd58df02d673d7070fcc599f18d19a43982a3932bed2c54f4f4d46c3a587950d37e8c8ef6d8f6be61614d3fb01eea11b3e54bc3baf424b59da1b77
+  metadata.gz: 2e56c03f7d5ef66c3583099ad2a23fe3cfeb2c61bab35af05eb9d8a48eaf845d2587bec39d9fbd90436b79e15d60df4b5d0495d0690eea37d651d6098e8943fb
+  data.tar.gz: 0e91ef1eca2eb5ce88a0f3fbef75824251bb121cc0f4396decc6e3def7a6877ea4bb8f48858a5ef58398117ec61c96797153d262e01d7d111285ea2ae0cef1aa

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.2.9
+  - Fix decoding of structs
+
 ## 1.2.8
   - Update protobuf library to 3.22.2
 

data/lib/logstash/codecs/protobuf.rb

@@ -2,6 +2,7 @@
 require 'logstash/codecs/base'
 require 'logstash/util/charset'
 require 'google/protobuf' # for protobuf3
+require 'google/protobuf/struct_pb'
 require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers, for protobuf2
 
 # Monkey-patch the `Google::Protobuf::DescriptorPool` with a mutex for exclusive
@@ -226,11 +227,14 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     end
     yield e if block_given?
   rescue => ex
-    @logger.warn("Couldn't decode protobuf: #{ex.inspect}.")
+    @logger.warn("Couldn't decode protobuf: #{ex.inspect}")
     if stop_on_error
       raise ex
     else # keep original message so that the user can debug it.
-      yield LogStash::Event.new("message" => data, "tags" => ["_protobufdecodefailure"])
+      yield LogStash::Event.new(
+        "message" => data, "tags" => ["_protobufdecodefailure"],
+        "decoder_exception" => "#{ex.inspect}"
+      )
     end
   end # def decode
 
@@ -250,9 +254,17 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   private
   def pb3_deep_to_hash(input)
     case input
+    when Google::Protobuf::Struct
+      result = JSON.parse input.to_json({
+        :preserve_proto_fieldnames => true,
+        :emit_defaults => true
+      })
     when Google::Protobuf::MessageExts # it's a protobuf class
       result = Hash.new
+      # when we've called to_h here it is already a nested hash, for the
+      # structs we bubble down the original value instead.
       input.to_h.each {|key, value|
+        value = input[key] if input[key].is_a? Google::Protobuf::Struct
         result[key] = pb3_deep_to_hash(value) # the key is required for the class lookup of enums.
       }
     when ::Array

data/logstash-codec-protobuf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '1.2.8'
+  s.version         = '1.2.9'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads protobuf messages and converts to Logstash Events"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/pb3_decode_spec.rb

@@ -76,7 +76,7 @@ describe LogStash::Codecs::Protobuf do
   context "#pb3decoder_test1" do
 
 
-    #### Test case 1: Decode simple protobuf ####################################################################################################################
+    #### Test case 1: Decode simple protobuf ##############################################
     let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new(
       "class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)
     }
@@ -84,8 +84,8 @@ describe LogStash::Codecs::Protobuf do
     it "should return an event from protobuf data" do
 
       unicorn_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
-      data = {:name => 'Pinkie', :age => 18, :is_pegasus => false, :favourite_numbers => [4711,23], :fur_colour => Colour::PINK,
-      :favourite_colours => [Colour::GREEN, Colour::BLUE]
+      data = {:name => 'Pinkie', :age => 18, :is_pegasus => false, :favourite_numbers => [4711,23],
+        :fur_colour => Colour::PINK, :favourite_colours => [Colour::GREEN, Colour::BLUE]
       }
 
       unicorn_object = unicorn_class.new(data)
@@ -103,8 +103,9 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test2" do
 
-    #### Test case 2: decode nested protobuf ####################################################################################################################
-    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)  }
+    #### Test case 2: decode nested protobuf ##############################################
+    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn",
+     "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)  }
 
     it "should return an event from protobuf data with nested classes" do
       father = unicorn_class.new({:name=> "Sparkle", :age => 50, :fur_colour => 3 })
@@ -126,8 +127,9 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test3" do
 
-    #### Test case 3: decode ProbeResult ####################################################################################################################
-    let(:plugin_3) { LogStash::Codecs::Protobuf.new("class_name" => "ProbeResult", "include_path" => [pb_include_path + '/pb3/ProbeResult_pb.rb'], "protobuf_version" => 3)  }
+    #### Test case 3: decode ProbeResult ##############################################
+    let(:plugin_3) { LogStash::Codecs::Protobuf.new("class_name" => "ProbeResult",
+     "include_path" => [pb_include_path + '/pb3/ProbeResult_pb.rb'], "protobuf_version" => 3)  }
 
     before do
         plugin_3.register
@@ -158,8 +160,9 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test4" do
 
-    #### Test case 4: decode PBDNSMessage ####################################################################################################################
-    let(:plugin_4) { LogStash::Codecs::Protobuf.new("class_name" => "PBDNSMessage", "include_path" => [pb_include_path + '/pb3/dnsmessage_pb.rb'], "protobuf_version" => 3)  }
+    #### Test case 4: decode PBDNSMessage ##############################################
+    let(:plugin_4) { LogStash::Codecs::Protobuf.new("class_name" => "PBDNSMessage",
+      "include_path" => [pb_include_path + '/pb3/dnsmessage_pb.rb'], "protobuf_version" => 3)  }
 
     before do
         plugin_4.register
@@ -167,7 +170,6 @@ describe LogStash::Codecs::Protobuf do
 
     it "should return an event from protobuf data with nested classes" do
 
-
       pbdns_message_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage").msgclass
       dns_question_class  = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSQuestion").msgclass
       dns_response_class  = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse").msgclass
@@ -239,7 +241,7 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test5" do
 
-    #### Test case 5: decode test case for github issue 17 ####################################################################################################################
+    #### Test case 5: decode test case for github issue 17 ##############################################
     let(:plugin_5) { LogStash::Codecs::Protobuf.new("class_name" => "com.foo.bar.IntegerTestMessage", "include_path" => [pb_include_path + '/pb3/integertest_pb.rb'], "protobuf_version" => 3)  }
 
     before do
@@ -308,7 +310,7 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test7" do
 
-    #### Test case 6: decode test case for github issue 17 ####################################################################################################################
+    #### Test case 6: decode test case for github issue 17 ##############################################
     let(:plugin_7) { LogStash::Codecs::Protobuf.new("class_name" => "RepeatedEvents", "include_path" => [pb_include_path + '/pb3/events_pb.rb'], "protobuf_version" => 3)  }
     before do
         plugin_7.register
@@ -338,7 +340,7 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test8a" do
 
-    ########################################################################################################################
+    ##################################################
     let(:plugin_8a) { LogStash::Codecs::Protobuf.new("class_name" => "FantasyHorse", "class_file" => 'pb3/FantasyHorse_pb.rb',
       "protobuf_root_directory" => pb_include_path, "protobuf_version" => 3, "pb3_set_oneof_metainfo" => true)  }
     before do
@@ -376,7 +378,7 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test8b" do
 
-    ########################################################################################################################
+    ##################################################
     let(:plugin_8b) { LogStash::Codecs::Protobuf.new("class_name" => "FantasyHorse", "class_file" => 'pb3/FantasyHorse_pb.rb',
       "protobuf_root_directory" => pb_include_path, "protobuf_version" => 3, "pb3_set_oneof_metainfo" => false)  }
     before do
@@ -410,7 +412,7 @@ describe LogStash::Codecs::Protobuf do
 
   context "#pb3decoder_test8c" do # same test as 8a just with different one_of options selected
 
-    ########################################################################################################################
+    ##################################################
     let(:plugin_8c) { LogStash::Codecs::Protobuf.new("class_name" => "FantasyHorse", "class_file" => 'pb3/FantasyHorse_pb.rb',
       "protobuf_root_directory" => pb_include_path, "protobuf_version" => 3, "pb3_set_oneof_metainfo" => true)  }
     before do
@@ -442,4 +444,33 @@ describe LogStash::Codecs::Protobuf do
 
   end # context pb3decoder_test8c
 
+  context "#pb3decoder_test9" do
+
+    ##################################################
+    let(:plugin_9) { LogStash::Codecs::Protobuf.new("class_name" => "messages.SendJsonRequest", "class_file" => 'pb3/struct_test_pb.rb',
+      "protobuf_root_directory" => pb_include_path, "protobuf_version" => 3, "pb3_set_oneof_metainfo" => false)  }
+      # TODO: pb3_set_oneof_metainfo => true is broken when running with a struct field
+    before do
+        plugin_9.register
+    end
+
+    require pb_include_path + '/pb3/struct_test_pb.rb'
+
+    it "should decode a message with an embedded struct" do
+      # nested struct field
+      details = Google::Protobuf::Struct.new(
+        fields: {"field_a" => {:string_value => "value_a"}},
+      )
+      data = {:UserID=>"123-456", :Details => details}
+      pb_obj = Messages::SendJsonRequest.new(data)
+      bin = Messages::SendJsonRequest.encode(pb_obj)
+
+      plugin_9.decode(bin) do |event|
+        expect(event.get("UserID") ).to eq(data[:UserID])
+        expect(event.get("Details") ).to eq({"field_a"=>"value_a"})
+      end
+
+    end # it
+  end # context pb3decoder_test9
+
 end # describe

data/spec/helpers/pb3/struct_test_pb.rb

@@ -0,0 +1,21 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+
+begin; require 'google/protobuf'; rescue LoadError; end
+
+begin; require 'google/protobuf/struct_pb'; rescue LoadError; end
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("RequestStruct.proto", :syntax => :proto3) do
+    add_message "messages.SendJsonRequest" do
+      optional :UserID, :string, 1
+      optional :Details, :message, 2, "google.protobuf.Struct"
+    end
+    add_message "messages.SendJsonResponse" do
+      optional :Response, :string, 1
+    end
+  end
+end
+
+module Messages
+  SendJsonRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("messages.SendJsonRequest").msgclass
+  SendJsonResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("messages.SendJsonResponse").msgclass
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-protobuf
 version: !ruby/object:Gem::Version
-  version: 1.2.8
+  version: 1.2.9
 platform: jruby
 authors:
 - Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-28 00:00:00.000000000 Z
+date: 2023-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -118,6 +118,7 @@ files:
 - spec/helpers/pb3/rum2_pb.rb
 - spec/helpers/pb3/rum3_pb.rb
 - spec/helpers/pb3/rum_pb.rb
+- spec/helpers/pb3/struct_test_pb.rb
 - spec/helpers/pb3/unicorn.proto3
 - spec/helpers/pb3/unicorn_pb.rb
 homepage:
@@ -141,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
+rubygems_version: 3.4.11
 signing_key:
 specification_version: 4
 summary: Reads protobuf messages and converts to Logstash Events
@@ -173,5 +174,6 @@ test_files:
 - spec/helpers/pb3/rum2_pb.rb
 - spec/helpers/pb3/rum3_pb.rb
 - spec/helpers/pb3/rum_pb.rb
+- spec/helpers/pb3/struct_test_pb.rb
 - spec/helpers/pb3/unicorn.proto3
 - spec/helpers/pb3/unicorn_pb.rb