logstash-codec-protobuf 1.0.5 → 1.1.0

data/logstash-codec-protobuf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '1.0.5'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads protobuf messages and converts to Logstash Events"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'google-protobuf', '3.1'
+  # s.add_runtime_dependency 'google-protobuf', '3.1'
+  s.add_runtime_dependency 'google-protobuf', '3.5.0.pre'
   s.add_runtime_dependency 'ruby-protocol-buffers' # for protobuf 2
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/codecs/protobuf3_spec.rb

@@ -10,12 +10,13 @@ require 'google/protobuf' # for protobuf3
 
 describe LogStash::Codecs::Protobuf do
 
+  pb_include_path = "../../../spec/helpers/"
 
-  context "#decodePB3" do
+  context "#test1_pb3" do
 
 
     #### Test case 1: Decode simple protobuf ####################################################################################################################
-    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => ['spec/helpers/pb3/unicorn_pb.rb'], "protobuf_version_3" => true)  }
+    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)  }
     before do
         plugin_unicorn.register      
     end
@@ -38,13 +39,15 @@ describe LogStash::Codecs::Protobuf do
         expect(event.get("is_pegasus") ).to eq(data[:is_pegasus] )
       end
     end # it
+  end # context 
 
+  context "#test2_pb3" do
 
 
 
 
     #### Test case 2: decode nested protobuf ####################################################################################################################
-    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => ['spec/helpers/pb3/unicorn_pb.rb'], "protobuf_version_3" => true)  }
+    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)  }
     before do
         plugin_unicorn.register      
     end
@@ -69,19 +72,150 @@ describe LogStash::Codecs::Protobuf do
       end
     end # it
 
+  end # context 
 
-  end # context #decodePB3
+  context "#test3_pb3" do
+
+    #### Test case 3: decode ProbeResult ####################################################################################################################
+    let(:plugin_3) { LogStash::Codecs::Protobuf.new("class_name" => "ProbeResult", "include_path" => [pb_include_path + '/pb3/ProbeResult_pb.rb'], "protobuf_version" => 3)  }
+    before do
+        plugin_3.register      
+    end
+
+    it "should return an event from protobuf encoded data with nested classes" do
+    
+
+      probe_result_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("ProbeResult").msgclass
+      ping_result_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PingIPv4Result").msgclass
+
+      ping_result_data = {:status=> PingIPv4Result::Status::ERROR, 
+        :latency => 50, :ip => "8.8.8.8", :probe_ip => "127.0.0.1", :geolocation => "New York City" }
+      ping_result_object = ping_result_class.new(ping_result_data)
+
+      probe_result_data = {:UUID => '12345678901233456789', :TaskPingIPv4Result => ping_result_object}
+      probe_result_object = probe_result_class.new(probe_result_data)
+      bin = probe_result_class.encode(probe_result_object)
+      plugin_3.decode(bin) do |event|
+        expect(event.get("UUID") ).to eq(probe_result_data[:UUID] )
+        expect(event.get("TaskPingIPv4Result")["status"] ).to eq("ERROR")
+        expect(event.get("TaskPingIPv4Result")["latency"] ).to eq(ping_result_data[:latency] )
+        expect(event.get("TaskPingIPv4Result")["ip"] ).to eq(ping_result_data[:ip] )
+        expect(event.get("TaskPingIPv4Result")["probe_ip"] ).to eq(ping_result_data[:probe_ip] )
+        expect(event.get("TaskPingIPv4Result")["geolocation"] ).to eq(ping_result_data[:geolocation] )
+      end
+    end # it
+  end # context 
+
+  context "#test4_pb3" do
+
+    #### Test case 4: decode PBDNSMessage ####################################################################################################################
+    let(:plugin_4) { LogStash::Codecs::Protobuf.new("class_name" => "PBDNSMessage", "include_path" => [pb_include_path + '/pb3/dnsmessage_pb.rb'], "protobuf_version" => 3)  }
+    before do
+        plugin_4.register      
+    end
+
+    it "should return an event from protobuf encoded data with nested classes" do
+    
+
+      pbdns_message_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage").msgclass
+      dns_question_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSQuestion").msgclass
+      dns_response_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse").msgclass
+      dns_rr_class       = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse.DNSRR").msgclass
+
+      dns_question_data = {:qName => "Foo", :qType => 12345, :qClass => 67890 }
+      dns_question_object = dns_question_class.new(dns_question_data)
+
+      dns_response_data = {:rcode => 12345, :appliedPolicy => "baz", :tags => ["a","b","c"], 
+        :queryTimeSec => 123, :queryTimeUsec => 456, 
+        :appliedPolicyType => PBDNSMessage::PolicyType::NSIP}
+
+      dns_rr_data = [
+        {:name => "abc", :type => 9000, :class => 8000, :ttl => 20, :rdata => "300"},
+        {:name => "def", :type => 19000, :class => 18000, :ttl => 120, :rdata => "1300"}
+      ]
+
+      dns_response_data[:rrs] = dns_rr_data.map { | d | d = dns_rr_class.new(d) }
+      dns_response_object = dns_response_class.new(dns_response_data)
+
+      pbdns_message_data = {
+        # :UUID => '12345678901233456789', :TaskPingIPv4Result => ping_result_object
+        :type => PBDNSMessage::Type::DNSIncomingResponseType,
+        :messageId => "15",
+        :serverIdentity => "16",
+        :socketFamily => PBDNSMessage::SocketFamily::INET6,
+        :socketProtocol => PBDNSMessage::SocketProtocol::TCP,
+        :from => "17",
+        :to => "18",
+        :inBytes => 70000,
+        :timeSec => 80000,
+        :timeUsec => 90000,
+        :id => 20000,
+        :question => dns_question_object,
+        :response => dns_response_object,
+        :originalRequestorSubnet => "19",
+        :requestorId => "Bar",
+        :initialRequestId => "20",
+        :deviceId => "21",
+        }
+      pbdns_message_object = pbdns_message_class.new(pbdns_message_data)
+      bin = pbdns_message_class.encode(pbdns_message_object)
+      plugin_4.decode(bin) do |event|
+        
+        ['messageId', 'serverIdentity','from','to','inBytes','timeUsec','timeSec','id', 'originalRequestorSubnet', 'requestorId' ,'initialRequestId','deviceIdf'].each { |n|  
+          expect(event.get(n)).to eq(pbdns_message_data[n.to_sym] ) }
+
+        # enum test: 
+        expect(event.get("type") ).to eq("DNSIncomingResponseType" )
+        expect(event.get("socketFamily") ).to eq("INET6" )
+        expect(event.get("socketProtocol") ).to eq("TCP" )
+
+        expect(event.get("question")["qName"] ).to eq(dns_question_data[:qName] )
+        expect(event.get("question")["qType"] ).to eq(dns_question_data[:qType] )
+        expect(event.get("question")["qClass"] ).to eq(dns_question_data[:qClass] )
+    
+        ['rcode', 'appliedPolicy','tags','queryTimeSec','queryTimeUsec'].each { |n|   expect(event.get('response')[n]).to eq(dns_response_data[n.to_sym] )   }
+        expect(event.get("response")['appliedPolicyType'] ).to eq("NSIP" )
+
+        dns_rr_data.each_with_index { | data, index | 
+          found = event.get("response")['rrs'][index]
+          ['name', 'type','class','ttl','rdata'].each { |n|   expect(found[n]).to eq(data[n.to_sym])   }
+        }
+
+      end
+    end # it
+
+  end # context 
+
+  context "#test5_pb3" do
+
+#### Test case 5: decode test case for github issue 17 ####################################################################################################################
+    let(:plugin_5) { LogStash::Codecs::Protobuf.new("class_name" => "com.foo.bar.IntegerTestMessage", "include_path" => [pb_include_path + '/pb3/integertest_pb.rb'], "protobuf_version" => 3)  }
+    before do
+        plugin_5.register      
+    end
+
+    it "should return an event from protobuf encoded data with nested classes" do
+      integertest_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("com.foo.bar.IntegerTestMessage").msgclass
+      integertest_object = integertest_class.new({:response_time => 500})
+      bin = integertest_class.encode(integertest_object)
+      plugin_5.decode(bin) do |event|
+        expect(event.get("response_time") ).to eq(500)
+      end
+    end # it
+
+
+  end # context
 
 
   context "#encodePB3-a" do
 
     #### Test case 3: encode simple protobuf ####################################################################################################################
 
-    definitions_file = 'spec/helpers/pb3/unicorn_pb.rb'
-    require definitions_file
+
+    require_relative '../helpers/pb3/unicorn_pb.rb'
 
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [definitions_file], "protobuf_version_3" => true)
+      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)
     end
 
     event3 = LogStash::Event.new("name" => "Pinkie", "age" => 18, "is_pegasus" => false, "favourite_numbers" => [1,2,3], "fur_colour" => Colour::PINK, "favourite_colours" => [1,5]      )
@@ -109,11 +243,10 @@ describe LogStash::Codecs::Protobuf do
 
     #### Test case 4: encode nested protobuf ####################################################################################################################
 
-    definitions_file = 'spec/helpers/pb3/unicorn_pb.rb'
-    require definitions_file
+    require_relative '../helpers/pb3/unicorn_pb.rb'
 
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [definitions_file], "protobuf_version_3" => true)
+      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)
     end
 
     event4 = LogStash::Event.new("name" => "Horst", "age" => 23, "is_pegasus" => true, "mother" => \

data/spec/codecs/protobuf_spec.rb

@@ -10,13 +10,15 @@ require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffer
 
 describe LogStash::Codecs::Protobuf do
 
+  pb_include_path = "../../../spec/helpers/"
 
 
-  context "#decodePB2" do
+  context "#test1" do
 
+    
 
     #### Test case 1: Decode simple protobuf bytes for unicorn ####################################################################################################################
-    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Unicorn", "include_path" => ['spec/helpers/pb2/unicorn.pb.rb'])  }
+    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Unicorn", "include_path" => [pb_include_path + '/pb2/unicorn.pb.rb'])  }
     before do
         plugin_unicorn.register      
     end
@@ -34,14 +36,14 @@ describe LogStash::Codecs::Protobuf do
       end
     end # it
 
-
+  end
 
     #### Test case 2: Decode complex protobuf bytes for human #####################################################################################################################
 
-
+  context "#test2" do
   
  
-    let(:plugin_human) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => ['spec/helpers/pb2/human.pb.rb'])  }
+    let(:plugin_human) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => [pb_include_path + '/pb2/human.pb.rb'])  }
     before do
         plugin_human.register      
     end
@@ -72,18 +74,15 @@ describe LogStash::Codecs::Protobuf do
         expect(event.get("[father][middle_names]") ).to eq(data_f[:middle_names] )
       end
     end # it
-
-
-
-
+  end # context
 
 
     #### Test case 3: Decoder test for enums #####################################################################################################################
 
-
+  context "#test3" do
   
  
-    let(:plugin_col) { LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/pb2/ColourTestcase.pb.rb'])  }
+    let(:plugin_col) { LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => [pb_include_path + '/pb2/ColourTestcase.pb.rb'])  }
     before do
         plugin_col.register      
     end
@@ -102,7 +101,7 @@ describe LogStash::Codecs::Protobuf do
     end # it
 
 
-  end # context decodePB2
+  end # context test3
 
 
 
@@ -112,7 +111,7 @@ describe LogStash::Codecs::Protobuf do
 
   context "#encodePB2-a" do
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::UnicornEvent", "include_path" => ['spec/helpers/pb2/unicorn_event.pb.rb']) 
+      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::UnicornEvent", "include_path" => [pb_include_path + '/pb2/unicorn_event.pb.rb']) 
     end
 
     event = LogStash::Event.new("colour" => "pink", "horn_length" => 12, "last_seen" => 1410081999, "has_wings" => true)    
@@ -140,7 +139,7 @@ describe LogStash::Codecs::Protobuf do
   
   context "#encodePB2-b" do
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => ['spec/helpers/pb2/human.pb.rb']) 
+      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => [pb_include_path + '/pb2/human.pb.rb']) 
     end
 
     event = LogStash::Event.new("first_name" => "Jimmy", "middle_names" => ["Bob", "James"], "last_name" => "Doe" \
@@ -180,10 +179,12 @@ describe LogStash::Codecs::Protobuf do
  
   context "#encodePB2-c" do
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/pb2/ColourTestcase.pb.rb'])
+      next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => [pb_include_path + '/pb2/ColourTestcase.pb.rb'])
     end
 
-    require 'spec/helpers/pb2/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
+    require_relative '../helpers/pb2/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
+    # ^ this import is run from the spec directory, $LOGSTASH_DIR/spec/codecs/
+    
     event = LogStash::Event.new("booleantest" =>  [false, false, true], "least_liked" => ColourProtoTest::Colour::YELLOW,  "favourite_colours" => \
        [ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )    
 

data/spec/helpers/pb3/ProbeResult_pb.rb

@@ -0,0 +1,26 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: results.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+ add_message "ProbeResult" do
+   optional :UUID, :string, 1
+   optional :TaskPingIPv4Result, :message, 2, "PingIPv4Result"
+ end
+ add_message "PingIPv4Result" do
+   optional :status, :enum, 2, "PingIPv4Result.Status"
+   optional :latency, :double, 3
+   optional :ip, :string, 4
+   optional :probe_ip, :string, 5
+   optional :geolocation, :string, 6
+ end
+ add_enum "PingIPv4Result.Status" do
+   value :OK, 0
+   value :ERROR, 1
+ end
+end
+
+ProbeResult = Google::Protobuf::DescriptorPool.generated_pool.lookup("ProbeResult").msgclass
+PingIPv4Result = Google::Protobuf::DescriptorPool.generated_pool.lookup("PingIPv4Result").msgclass
+PingIPv4Result::Status = Google::Protobuf::DescriptorPool.generated_pool.lookup("PingIPv4Result.Status").enummodule

data/spec/helpers/pb3/dnsmessage_pb.rb

@@ -0,0 +1,82 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: dnsmessage.proto3
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "PBDNSMessage" do
+    optional :type, :enum, 1, "PBDNSMessage.Type"
+    optional :messageId, :bytes, 2
+    optional :serverIdentity, :bytes, 3
+    optional :socketFamily, :enum, 4, "PBDNSMessage.SocketFamily"
+    optional :socketProtocol, :enum, 5, "PBDNSMessage.SocketProtocol"
+    optional :from, :bytes, 6
+    optional :to, :bytes, 7
+    optional :inBytes, :uint64, 8
+    optional :timeSec, :uint32, 9
+    optional :timeUsec, :uint32, 10
+    optional :id, :uint32, 11
+    optional :question, :message, 12, "PBDNSMessage.DNSQuestion"
+    optional :response, :message, 13, "PBDNSMessage.DNSResponse"
+    optional :originalRequestorSubnet, :bytes, 14
+    optional :requestorId, :string, 15
+    optional :initialRequestId, :bytes, 16
+    optional :deviceId, :bytes, 17
+  end
+  add_message "PBDNSMessage.DNSQuestion" do
+    optional :qName, :string, 1
+    optional :qType, :uint32, 2
+    optional :qClass, :uint32, 3
+  end
+  add_message "PBDNSMessage.DNSResponse" do
+    optional :rcode, :uint32, 1
+    repeated :rrs, :message, 2, "PBDNSMessage.DNSResponse.DNSRR"
+    optional :appliedPolicy, :string, 3
+    repeated :tags, :string, 4
+    optional :queryTimeSec, :uint32, 5
+    optional :queryTimeUsec, :uint32, 6
+    optional :appliedPolicyType, :enum, 7, "PBDNSMessage.PolicyType"
+  end
+  add_message "PBDNSMessage.DNSResponse.DNSRR" do
+    optional :name, :string, 1
+    optional :type, :uint32, 2
+    optional :class, :uint32, 3
+    optional :ttl, :uint32, 4
+    optional :rdata, :bytes, 5
+  end
+  add_enum "PBDNSMessage.Type" do
+    value :Dummy0, 0
+    value :DNSQueryType, 1
+    value :DNSResponseType, 2
+    value :DNSOutgoingQueryType, 3
+    value :DNSIncomingResponseType, 4
+  end
+  add_enum "PBDNSMessage.SocketFamily" do
+    value :Dummy1, 0
+    value :INET, 1
+    value :INET6, 2
+  end
+  add_enum "PBDNSMessage.SocketProtocol" do
+    value :Dummy2, 0
+    value :UDP, 1
+    value :TCP, 2
+  end
+  add_enum "PBDNSMessage.PolicyType" do
+    value :Dummy3, 0
+    value :UNKNOWN, 1
+    value :QNAME, 2
+    value :CLIENTIP, 3
+    value :RESPONSEIP, 4
+    value :NSDNAME, 5
+    value :NSIP, 6
+  end
+end
+
+PBDNSMessage = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage").msgclass
+PBDNSMessage::DNSQuestion = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSQuestion").msgclass
+PBDNSMessage::DNSResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse").msgclass
+PBDNSMessage::DNSResponse::DNSRR = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse.DNSRR").msgclass
+PBDNSMessage::Type = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.Type").enummodule
+PBDNSMessage::SocketFamily = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.SocketFamily").enummodule
+PBDNSMessage::SocketProtocol = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.SocketProtocol").enummodule
+PBDNSMessage::PolicyType = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.PolicyType").enummodule

data/spec/helpers/pb3/integertest_pb.rb

@@ -0,0 +1,20 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: TestMessage.proto3
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "com.foo.bar.IntegerTestMessage" do
+    optional :response_time, :int64, 1
+  end
+end
+
+module Com
+  module Bla
+    module Bla
+      module Bla
+        TestMessage = Google::Protobuf::DescriptorPool.generated_pool.lookup("com.foo.bar.IntegerTestMessage").msgclass
+      end
+    end
+  end
+end