logstash-codec-protobuf 1.0.5 → 1.2.5

data/spec/codecs/pb3_encode_spec.rb

@@ -0,0 +1,243 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/protobuf"
+require "logstash/event"
+
+require 'google/protobuf' # for protobuf3
+
+# absolute path to the protobuf helpers directory
+pb_include_path = File.expand_path(".") + "/spec/helpers"
+
+describe LogStash::Codecs::Protobuf do
+
+  context "#encodePB3-a" do
+
+    #### Test case 1: encode simple protobuf ####################################################################################################################
+
+    require_relative '../helpers/pb3/unicorn_pb.rb'
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)
+    end
+
+    event1 = LogStash::Event.new("name" => "Pinkie", "age" => 18, "is_pegasus" => false, "favourite_numbers" => [1,2,3], "fur_colour" => Colour::PINK, "favourite_colours" => [1,5] )
+
+    it "should return protobuf encoded data for testcase 1" do
+
+      subject.on_event do |event, data|
+        expect(data).to be_a(String)
+
+        pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
+        decoded_data = pb_builder.decode(data)
+        expect(decoded_data.name ).to eq(event.get("name") )
+        expect(decoded_data.age ).to eq(event.get("age") )
+        expect(decoded_data.is_pegasus ).to eq(event.get("is_pegasus") )
+        expect(decoded_data.fur_colour ).to eq(:PINK)
+        expect(decoded_data.favourite_numbers ).to eq(event.get("favourite_numbers") )
+        expect(decoded_data.favourite_colours ).to eq([:BLUE,:WHITE] )
+      end # subject.on_event
+
+      subject.encode(event1)
+    end # it
+
+  end # context
+
+  context "#encodePB3-b" do
+
+    #### Test case 2: encode nested protobuf ####################################################################################################################
+
+    require_relative '../helpers/pb3/unicorn_pb.rb'
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [pb_include_path + '/pb3/unicorn_pb.rb'], "protobuf_version" => 3)
+    end
+
+    event = LogStash::Event.new("name" => "Horst", "age" => 23, "is_pegasus" => true, "mother" => \
+        {"name" => "Mom", "age" => 47}, "father" => {"name"=> "Daddy", "age"=> 50, "fur_colour" => 3 } # 3 == SILVER
+      )
+
+    it "should return protobuf encoded data for testcase 2" do
+
+      subject.on_event do |event, data|
+        expect(data).to be_a(String)
+
+        pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
+        decoded_data = pb_builder.decode(data)
+
+        expect(decoded_data.name ).to eq(event.get("name") )
+        expect(decoded_data.age ).to eq(event.get("age") )
+        expect(decoded_data.is_pegasus ).to eq(event.get("is_pegasus") )
+        expect(decoded_data.mother.name ).to eq(event.get("mother")["name"] )
+        expect(decoded_data.mother.age ).to eq(event.get("mother")["age"] )
+        expect(decoded_data.father.name ).to eq(event.get("father")["name"] )
+        expect(decoded_data.father.age ).to eq(event.get("father")["age"] )
+        expect(decoded_data.father.fur_colour ).to eq(:SILVER)
+
+
+      end # subject4.on_event
+      subject.encode(event)
+    end # it
+
+  end # context #encodePB3
+
+  context "encodePB3-c" do
+
+    #### Test case 3: encode nested protobuf ####################################################################################################################
+
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "something.rum_akamai.ProtoAkamaiRum", "include_path" => [pb_include_path + '/pb3/rum_pb.rb' ], "protobuf_version" => 3)
+    end
+
+    event = LogStash::Event.new(
+      "user_agent"=>{"os"=>"Android OS", "family"=>"Chrome Mobile", "major"=>74, "mobile"=>"1", "minor"=>0, "manufacturer"=>"Samsung", "osversion"=>"8",
+          "model"=>"Galaxy S7 Edge", "type"=>"Mobile",
+          "raw"=>"Mozilla/5.0 (Linux; Android 8.0.0; SM-G935F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36"},
+          "dom"=>{"script"=>65, "ln"=>2063, "ext"=>47}, "page_group"=>"SEO-Pages",
+          "active_ctests"=>["1443703219", "47121", "47048", "46906"], "timestamp"=>"1559566982508",
+          "geo"=>{"isp"=>"Telecom Italia Mobile", "lat"=>45.4643, "postalcode"=>"20123", "netspeed"=>"Cellular", "rg"=>"MI", "cc"=>"IT",
+            "organisation"=>"Telecom Italia Mobile", "ovr"=>false, "city"=>"Milan", "lon"=>9.1895},
+          "header"=>{"sender_id"=>"0"}, "domain"=>"something.com", "url"=>"https://www.something.it/",
+          "timers"=>{"tti"=>4544, "ttvr"=>3657, "fcp"=>2683, "ttfi"=>4280, "fid"=>31, "longtasks"=>2519, "t_resp"=>1748}
+    )
+
+    it "should return protobuf encoded data for testcase 3" do
+
+      subject.on_event do |event, data|
+        expect(data).to be_a(String)
+
+        pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("something.rum_akamai.ProtoAkamaiRum").msgclass
+        decoded_data = pb_builder.decode(data)
+
+        expect(decoded_data.domain ).to eq(event.get("domain") )
+        expect(decoded_data.dom.ext ).to eq(event.get("dom")["ext"] )
+        expect(decoded_data.user_agent.type ).to eq(event.get("user_agent")["type"] )
+        expect(decoded_data.geo.rg ).to eq(event.get("geo")["rg"] )
+
+
+      end # subject4.on_event
+      subject.encode(event)
+    end # it
+  end # context #encodePB3-c
+
+
+  context "encodePB3-d" do
+
+    #### Test case 3: autoconvert data types ####################################################################################################################
+
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "something.rum_akamai.ProtoAkamai2Rum",
+        "pb3_encoder_autoconvert_types" => true,
+        "include_path" => [pb_include_path + '/pb3/rum2_pb.rb' ], "protobuf_version" => 3)
+    end
+
+    event = LogStash::Event.new(
+
+     # major should autoconvert to float
+     "user_agent"=>{"minor"=>0,"major"=>"74"},
+
+     # ext should autoconvert to int. script being empty should be ignored.
+     "dom"=>{"script"=>nil, "ln"=>2063, "ext"=>47.0},
+
+     # ovr should autoconvert to Boolean
+     "geo"=>{"ovr"=>"false"},
+
+     # sender_id should autoconvert to string
+     "header"=>{"sender_id"=>1},
+     "domain" => "www",
+
+     # should autoconvert to string
+     "http_referer" => 1234,
+    )
+
+
+    it "should fix datatypes to match the protobuf definition" do
+
+      subject.on_event do |event, data|
+        expect(data).to be_a(String)
+
+        pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("something.rum_akamai.ProtoAkamai2Rum").msgclass
+        decoded_data = pb_builder.decode(data)
+        expect(decoded_data.domain ).to eq(event.get("domain") )
+        expect(decoded_data.user_agent.major).to eq(74)
+        expect(decoded_data.dom.ext).to eq(47)
+        expect(decoded_data.geo.ovr).to eq(false)
+        expect(decoded_data.header.sender_id).to eq("1")
+        expect(decoded_data.http_referer).to eq("1234")
+
+      end
+      subject.encode(event)
+    end # it
+
+  end # context #encodePB3-d
+
+
+context "encodePB3-e" do
+
+    #### Test case 4: handle nil data ####################################################################################################################
+
+
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "something.rum_akamai.ProtoAkamai3Rum",
+        "pb3_encoder_autoconvert_types" => false,
+        "include_path" => [pb_include_path + '/pb3/rum3_pb.rb' ], "protobuf_version" => 3)
+    end
+
+    event = LogStash::Event.new(
+      "domain" => nil,
+      "header" => {"sender_id" => "23"},
+      "geo"=>{"organisation"=>"Jio", "rg"=>"DL", "netspeed"=>nil, "city"=>nil, "cc"=>"IN", "ovr"=>false, "postalcode"=>"110012", "isp"=>"Jio"}
+    )
+
+    it "should ignore empty fields" do
+
+      subject.on_event do |event, data|
+        expect(data).to be_a(String)
+
+        pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("something.rum_akamai.ProtoAkamai3Rum").msgclass
+        decoded_data = pb_builder.decode(data)
+        expect(decoded_data.geo.organisation ).to eq(event.get("geo")["organisation"])
+        expect(decoded_data.geo.ovr ).to eq(event.get("geo")["ovr"])
+        expect(decoded_data.geo.postalcode ).to eq(event.get("geo")["postalcode"])
+        expect(decoded_data.header.sender_id ).to eq(event.get("header")['sender_id'] )
+
+      end
+      subject.encode(event)
+    end # it
+
+  end # context #encodePB3-e
+
+
+
+context "encodePB3-f" do
+
+    #### Test case 5: handle additional fields (discard event without crashing pipeline) ####################################################################################################################
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "something.rum_akamai.ProtoAkamai3Rum",
+        "pb3_encoder_autoconvert_types" => false,
+        "include_path" => [pb_include_path + '/pb3/rum3_pb.rb' ], "protobuf_version" => 3)
+    end
+
+    event = LogStash::Event.new(
+      "domain" => nil, "bot" => "This field does not exist in the protobuf definition",
+      "header" => {"sender_id" => "23"},
+      "geo"=>{"organisation"=>"Jio", "rg"=>"DL", "netspeed"=>nil, "city"=>nil, "cc"=>"IN", "ovr"=>false, "postalcode"=>"110012", "isp"=>"Jio"}
+    )
+
+    it "should not return data" do
+
+      subject.on_event do |event, data|
+        expect("the on_event method should not be called").to eq("so this code should never be reached")
+      end
+      subject.encode(event)
+    end # it
+
+  end # context #encodePB3-f
+
+
+
+
+end # describe

data/spec/helpers/pb2/event.pb.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+require 'protocol_buffers'
+
+begin; require 'header/header.pb'; rescue LoadError; end
+
+module Logging
+  # forward declarations
+  class Event < ::ProtocolBuffers::Message; end
+
+  class Event < ::ProtocolBuffers::Message
+    set_fully_qualified_name "Logging.Event"
+
+    optional :string, :name, 1
+    optional ::Grpc::Header, :header, 2
+  end
+
+end

data/spec/helpers/pb2/event.proto

@@ -0,0 +1,12 @@
+syntax = "proto2";
+
+// Compile: ruby-protoc spec/helpers/*proto
+
+package Logging;
+
+import "header/header.proto";
+
+message Event {
+    optional string name = 1;
+    optional grpc.Header header = 2;
+}

data/spec/helpers/pb2/header/header.pb.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+require 'protocol_buffers'
+
+module Grpc
+  # forward declarations
+  class Header < ::ProtocolBuffers::Message; end
+
+  class Header < ::ProtocolBuffers::Message
+    set_fully_qualified_name "grpc.Header"
+
+    optional :string, :protocol, 1
+  end
+
+end

data/spec/helpers/pb2/header/header.proto

@@ -0,0 +1,8 @@
+syntax = "proto2";
+
+// Compile: ruby-protoc spec/helpers/*proto
+package grpc;
+
+message Header {
+    optional string protocol = 1;
+}

data/spec/helpers/pb3/FantasyHorse_pb.rb

@@ -0,0 +1,44 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+
+begin; require 'google/protobuf'; rescue LoadError; end
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "FantasyHorse" do
+    optional :name, :string, 1
+    oneof :horse_type do
+      optional :unicorn, :message, 2, "FantasyUnicorn"
+      optional :pegasus, :message, 3, "FantasyPegasus"
+    end
+    optional :tail, :message, 4, "FantasyHorseTail"
+  end
+  add_message "FantasyUnicorn" do
+    optional :horn_length, :int32, 1
+    optional :horn_colour, :string, 2
+  end
+  add_message "FantasyPegasus" do
+    optional :wings_length, :int32, 1
+    optional :wings_width, :int32, 2
+    optional :wings_feathers, :string, 3
+  end
+  add_message "FantasyHorseTail" do
+    optional :tail_length, :int32, 1
+    oneof :hair_type do
+      optional :braided, :message, 2, "BraidedHorseTail"
+      optional :natural, :message, 3, "NaturalHorseTail"
+    end
+  end
+  add_message "BraidedHorseTail" do
+    optional :braiding_style, :string, 1
+    optional :braid_thickness, :int32, 2
+  end
+  add_message "NaturalHorseTail" do
+    optional :wavyness, :string, 1
+  end
+end
+
+FantasyHorse = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyHorse").msgclass
+FantasyUnicorn = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyUnicorn").msgclass
+FantasyPegasus = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyPegasus").msgclass
+FantasyHorseTail = Google::Protobuf::DescriptorPool.generated_pool.lookup("FantasyHorseTail").msgclass
+BraidedHorseTail = Google::Protobuf::DescriptorPool.generated_pool.lookup("BraidedHorseTail").msgclass
+NaturalHorseTail = Google::Protobuf::DescriptorPool.generated_pool.lookup("NaturalHorseTail").msgclass

data/spec/helpers/pb3/ProbeResult_pb.rb

@@ -0,0 +1,26 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: results.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+ add_message "ProbeResult" do
+   optional :UUID, :string, 1
+   optional :TaskPingIPv4Result, :message, 2, "PingIPv4Result"
+ end
+ add_message "PingIPv4Result" do
+   optional :status, :enum, 2, "PingIPv4Result.Status"
+   optional :latency, :double, 3
+   optional :ip, :string, 4
+   optional :probe_ip, :string, 5
+   optional :geolocation, :string, 6
+ end
+ add_enum "PingIPv4Result.Status" do
+   value :OK, 0
+   value :ERROR, 1
+ end
+end
+
+ProbeResult = Google::Protobuf::DescriptorPool.generated_pool.lookup("ProbeResult").msgclass
+PingIPv4Result = Google::Protobuf::DescriptorPool.generated_pool.lookup("PingIPv4Result").msgclass
+PingIPv4Result::Status = Google::Protobuf::DescriptorPool.generated_pool.lookup("PingIPv4Result.Status").enummodule

data/spec/helpers/pb3/dnsmessage_pb.rb

@@ -0,0 +1,82 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: dnsmessage.proto3
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "PBDNSMessage" do
+    optional :type, :enum, 1, "PBDNSMessage.Type"
+    optional :messageId, :bytes, 2
+    optional :serverIdentity, :bytes, 3
+    optional :socketFamily, :enum, 4, "PBDNSMessage.SocketFamily"
+    optional :socketProtocol, :enum, 5, "PBDNSMessage.SocketProtocol"
+    optional :from, :bytes, 6
+    optional :to, :bytes, 7
+    optional :inBytes, :uint64, 8
+    optional :timeSec, :uint32, 9
+    optional :timeUsec, :uint32, 10
+    optional :id, :uint32, 11
+    optional :question, :message, 12, "PBDNSMessage.DNSQuestion"
+    optional :response, :message, 13, "PBDNSMessage.DNSResponse"
+    optional :originalRequestorSubnet, :bytes, 14
+    optional :requestorId, :string, 15
+    optional :initialRequestId, :bytes, 16
+    optional :deviceId, :bytes, 17
+  end
+  add_message "PBDNSMessage.DNSQuestion" do
+    optional :qName, :string, 1
+    optional :qType, :uint32, 2
+    optional :qClass, :uint32, 3
+  end
+  add_message "PBDNSMessage.DNSResponse" do
+    optional :rcode, :uint32, 1
+    repeated :rrs, :message, 2, "PBDNSMessage.DNSResponse.DNSRR"
+    optional :appliedPolicy, :string, 3
+    repeated :tags, :string, 4
+    optional :queryTimeSec, :uint32, 5
+    optional :queryTimeUsec, :uint32, 6
+    optional :appliedPolicyType, :enum, 7, "PBDNSMessage.PolicyType"
+  end
+  add_message "PBDNSMessage.DNSResponse.DNSRR" do
+    optional :name, :string, 1
+    optional :type, :uint32, 2
+    optional :class, :uint32, 3
+    optional :ttl, :uint32, 4
+    optional :rdata, :bytes, 5
+  end
+  add_enum "PBDNSMessage.Type" do
+    value :Dummy0, 0
+    value :DNSQueryType, 1
+    value :DNSResponseType, 2
+    value :DNSOutgoingQueryType, 3
+    value :DNSIncomingResponseType, 4
+  end
+  add_enum "PBDNSMessage.SocketFamily" do
+    value :Dummy1, 0
+    value :INET, 1
+    value :INET6, 2
+  end
+  add_enum "PBDNSMessage.SocketProtocol" do
+    value :Dummy2, 0
+    value :UDP, 1
+    value :TCP, 2
+  end
+  add_enum "PBDNSMessage.PolicyType" do
+    value :Dummy3, 0
+    value :UNKNOWN, 1
+    value :QNAME, 2
+    value :CLIENTIP, 3
+    value :RESPONSEIP, 4
+    value :NSDNAME, 5
+    value :NSIP, 6
+  end
+end
+
+PBDNSMessage = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage").msgclass
+PBDNSMessage::DNSQuestion = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSQuestion").msgclass
+PBDNSMessage::DNSResponse = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse").msgclass
+PBDNSMessage::DNSResponse::DNSRR = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.DNSResponse.DNSRR").msgclass
+PBDNSMessage::Type = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.Type").enummodule
+PBDNSMessage::SocketFamily = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.SocketFamily").enummodule
+PBDNSMessage::SocketProtocol = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.SocketProtocol").enummodule
+PBDNSMessage::PolicyType = Google::Protobuf::DescriptorPool.generated_pool.lookup("PBDNSMessage.PolicyType").enummodule