logstash-codec-protobuf 1.0.3 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87e516dce982aa8ec7ca0677d4ebc7fb5149d029
-  data.tar.gz: 42a1c2ce5dfe0b768c67629f5d7449d6bc423519
+  metadata.gz: a623045ad79469eeefd501735a5fc3284c19eef3
+  data.tar.gz: 1a4b69f9ae131f5abe2b55270073c975edcbeab3
 SHA512:
-  metadata.gz: 9e337bc75f2d2f2de21ef9e60be48f3ef87d695d1bad0cc4e4adfc6ec9733167a43ffbc19c68c6209259e8bd34a0b2206fed846a4c0874cd2738e05707aa77d2
-  data.tar.gz: 93c3a97f8537b31f0b819ad4ef900a0935c586dc5460b04e5aba486e94aa96fd5d247321e3b6d5a8b95a1034a6a5162e0cc92b555b4fa9ccaa248c9fc0a72c40
+  metadata.gz: 0e152a7b9675dbd1c430da257a340364944f71b430bd093913705f376971650cc46933395a492f38732dc3665472ca56ebdc3e995d0c5023236b67466a7dc835
+  data.tar.gz: a55114ffbc512e1ee31748093f864c1683e33536f590ded503cbae5ca2ec4f61754e7d518efc722593c9af4040270438f40fa05d3fa03ae27e334d1a61ff520c

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.0.5
+  - Add support for protobuf3
+
+## 1.0.4
+  - Update gemspec summary
+
 ## 1.0.3
   - Fix some documentation issues
 

data/LICENSE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright (c) 2012–2016 Elasticsearch http://www.elastic.co
+   Copyright (c) 2012-2018 Elasticsearch http://www.elastic.co
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/README.md

@@ -5,9 +5,7 @@ This is a codec plugin for [Logstash](https://github.com/elastic/logstash) to pa
 # Prerequisites and Installation
  
 * prepare your ruby versions of the protobuf definitions, for example using the ruby-protoc compiler from https://github.com/codekitchen/ruby-protocol-buffers
-* download the [gem file](https://rubygems.org/gems/logstash-codec-protobuf) to your computer.
-* Install the plugin. From within your logstash directory, do
-	bin/plugin install /path/to/logstash-codec-protobuf-$VERSION.gem
+* install the codec: `bin/logstash-plugin install logstash-codec-protobuf`
 * use the codec in your logstash config file. See details below.
 
 ## Configuration

data/lib/logstash/codecs/protobuf.rb

@@ -1,7 +1,8 @@
 # encoding: utf-8
 require 'logstash/codecs/base'
 require 'logstash/util/charset'
-require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers
+require 'google/protobuf' # for protobuf3
+require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers, for protobuf2
 
 # This codec converts protobuf encoded messages into logstash events and vice versa. 
 #
@@ -65,102 +66,255 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   #  
   config :include_path, :validate => :array, :required => true
 
+  # Protocol buffer version switch. Set to false (default) for version 2. Please note that the behaviour for enums varies between the versions. 
+  # For protobuf 2 you will get integer representations for enums, for protobuf 3 you'll get string representations due to a different converter library.
+  # Recommendation: use the translate plugin to restore previous behaviour when upgrading.
+  config :protobuf_version_3, :validate => :boolean, :required => true, :default=>false
+
 
   def register
-    @pb_metainfo = {}
-    include_path.each { |path| require_pb_path(path) }
-    @obj = create_object_from_name(class_name)
-    @logger.debug("Protobuf files successfully loaded.")
+    @metainfo_messageclasses = {}
+    @metainfo_enumclasses = {}
+    include_path.each { |path| load_protobuf_definition(path) }
+    if @protobuf_version_3      
+      @pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
+    else
+      @pb_builder = pb2_create_instance(class_name)
+    end
   end
 
 
   def decode(data)
     begin
-      decoded = @obj.parse(data.to_s)
-      yield LogStash::Event.new(decoded.to_hash) if block_given?
+      if @protobuf_version_3
+        decoded = @pb_builder.decode(data.to_s)
+        h = pb3_deep_to_hash(decoded)
+      else
+        decoded = @pb_builder.parse(data.to_s)
+        h = decoded.to_hash        
+      end
+      yield LogStash::Event.new(h) if block_given?
     rescue => e
       @logger.warn("Couldn't decode protobuf: #{e.inspect}.")
-      # raise e
+      raise e
     end
   end # def decode
 
 
   def encode(event)
-    protobytes = generate_protobuf(event)
-    @on_event.call(event, protobytes)
+    if @protobuf_version_3
+      protobytes = pb3_encode_wrapper(event)
+    else
+      protobytes = pb2_encode_wrapper(event)     
+    end
+     @on_event.call(event, protobytes)
   end # def encode
 
 
   private
-  def generate_protobuf(event)
+  def pb3_deep_to_hash(input)
+    if input.class.ancestors.include? Google::Protobuf::MessageExts # it's a protobuf class
+      result = Hash.new
+      input.to_hash.each {|key, value|
+        result[key] = pb3_deep_to_hash(value) # the key is required for the class lookup of enums.
+      }      
+    elsif input.kind_of?(Array)
+      result = []
+      input.each {|value|
+          result << pb3_deep_to_hash(value)
+      }
+    elsif input.kind_of?(::Hash)
+      result = {}
+      input.each {|key, value|
+          result[key] = pb3_deep_to_hash(value)
+      }
+    elsif input.instance_of? Symbol # is an Enum
+      result = input.to_s.sub(':','')
+    else
+      result = input
+    end
+    result
+  end
+
+  def pb3_encode_wrapper(event)
     begin
-      data = _encode(event, @class_name)
-      msg = @obj.new(data)
-      msg.serialize_to_string
-    rescue NoMethodError
-      @logger.debug("error 2: NoMethodError. Maybe mismatching protobuf definition. Required fields are: " + event.to_hash.keys.join(", "))
+      data = pb3_encode(event.to_hash, @class_name)
+      pb_obj = @pb_builder.new(data)
+      @pb_builder.encode(pb_obj)
+    rescue ArgumentError => e
+      @logger.debug("Encoding error 2. Probably mismatching protobuf definition. Required fields in the protobuf definition are: " + event.to_hash.keys.join(", ") + " and the timestamp field name must not include a @. ")
+      raise e
     rescue => e
       @logger.debug("Couldn't generate protobuf: ${e}")
+      raise e
     end
   end
 
 
-  def _encode(datahash, class_name)
-    fields = prepare_for_encoding(datahash)    
-    meta = get_complex_types(class_name) # returns a hash with member names and their protobuf class names
-    meta.map do | (k,typeinfo) |
-      if fields.include?(k)
-        original_value = fields[k] 
-        proto_obj = create_object_from_name(typeinfo)
-        fields[k] = 
+  def pb3_encode(datahash, class_name)
+    next unless datahash.is_a?(::Hash)
+
+    # Preparation: the data cannot be encoded until certain criteria are met:
+    # 1) remove @ signs from keys.
+    # 2) convert timestamps and other objects to strings
+    datahash = datahash.inject({}){|x,(k,v)| x[k.gsub(/@/,'').to_sym] = (should_convert_to_string?(v) ? v.to_s : v); x}
+    
+    # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
+    meta = @metainfo_messageclasses[class_name]
+    if meta
+      meta.map do | (field_name,class_name) |
+        key = field_name.to_sym
+        if datahash.include?(key)
+          original_value = datahash[key] 
+          datahash[key] = 
+            if original_value.is_a?(::Array)
+              # make this field an array/list of protobuf objects
+              # value is a list of hashed complex objects, each of which needs to be protobuffed and
+              # put back into the list.
+              original_value.map { |x| pb3_encode(x, class_name) } 
+              original_value
+            else 
+              r = pb3_encode(original_value, class_name)
+              builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
+              builder.new(r)              
+            end # if is array
+        end # if datahash_include
+      end # do
+    end # if meta
+    # Check if any of the fields in this hash are enum classes and if so, create a builder for them.
+    meta = @metainfo_enumclasses[class_name]
+    if meta
+      meta.map do | (field_name,class_name) |
+        key = field_name.to_sym
+        if datahash.include?(key)
+          original_value = datahash[key]
+          datahash[key] = 
           if original_value.is_a?(::Array)
-            # make this field an array/list of protobuf objects
-            # value is a list of hashed complex objects, each of which needs to be protobuffed and
-            # put back into the list.
-            original_value.map { |x| _encode(x, typeinfo) } 
+            original_value.map { |x| pb3_encode(x, class_name) } 
             original_value
-          else 
-            recursive_fix = _encode(original_value, class_name)
-            proto_obj.new(recursive_fix)
+          else
+            if original_value.is_a?(Fixnum)
+              original_value # integers will be automatically converted into enum
+            else
+              # feature request: support for providing integers as strings or symbols.
+              # not fully tested yet:
+              # begin
+              #   enum_lookup_name = "#{class_name}::#{original_value}"
+              #   enum_lookup_name.split('::').inject(Object) do |mod, class_name|
+              #     mod.const_get(class_name)
+              #   end # do
+              # rescue => e
+              #   @logger.debug("Encoding error 3: could not translate #{original_value} into enum. ${e}")
+              #   raise e
+              # end         
+            end # if is a fixnum    
           end # if is array
-      end
-    end    
-    fields
+        end # if datahash_include
+      end # do
+    end # if meta
+    datahash
+  end
+
+  def pb2_encode_wrapper(event)
+    begin
+      data = pb2_encode(event.to_hash, @class_name)
+      msg = @pb_builder.new(data)
+      msg.serialize_to_string
+    rescue NoMethodError => e
+      @logger.debug("Encoding error 2. Probably mismatching protobuf definition. Required fields in the protobuf definition are: " + event.to_hash.keys.join(", ") + " and the timestamp field name must not include a @. ")
+      raise e
+    rescue => e
+      @logger.debug("Encoding error 1: ${e}")
+      raise e
+    end
   end
 
 
-  def prepare_for_encoding(datahash)
-    # the data cannot be encoded until certain criteria are met:
-    # 1) remove @ signs from keys 
+  def pb2_encode(datahash, class_name)
+    next unless datahash.is_a?(::Hash)
+
+    # Preparation: the data cannot be encoded until certain criteria are met:
+    # 1) remove @ signs from keys.
     # 2) convert timestamps and other objects to strings
-    next unless datahash.is_a?(::Hash)    
-    ::Hash[datahash.map{|(k,v)| [remove_atchar(k.to_s), (convert_to_string?(v) ? v.to_s : v)] }]
+    datahash = ::Hash[datahash.map{|(k,v)| [k.to_s.dup.gsub(/@/,''), (should_convert_to_string?(v) ? v.to_s : v)] }]
+    
+    # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
+    meta = @metainfo_messageclasses[class_name]
+    if meta
+      meta.map do | (k,class_name) |
+        if datahash.include?(k)
+          original_value = datahash[k] 
+          p
+          datahash[k] = 
+            if original_value.is_a?(::Array)
+              # make this field an array/list of protobuf objects
+              # value is a list of hashed complex objects, each of which needs to be protobuffed and
+              # put back into the list.
+              original_value.map { |x| pb2_encode(x, class_name) } 
+              original_value
+            else 
+              proto_obj = pb2_create_instance(class_name)
+              proto_obj.new(pb2_encode(original_value, class_name))
+            end # if is array
+        end # if datahash_include
+      end # do
+    end # if meta
+
+    datahash
   end
 
 
-  def convert_to_string?(v)
+  def should_convert_to_string?(v)
     !(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
   end
 
-   
-  def remove_atchar(key) # necessary for @timestamp fields and the likes. Protobuf definition doesn't handle @ in field names well.
-    key.dup.gsub(/@/,'')
-  end
-
   
-  def create_object_from_name(name)
+  def pb2_create_instance(name)
     begin
       @logger.debug("Creating instance of " + name)
       name.split('::').inject(Object) { |n,c| n.const_get c }
      end
   end
 
-  def get_complex_types(class_name)
-    @pb_metainfo[class_name]
+
+  def pb3_metadata_analyis(filename)
+    regex_class_name = /\s*add_message "(?<name>.+?)" do\s+/ # TODO optimize both regexes for speed (negative lookahead)
+    regex_pbdefs = /\s*(optional|repeated)(\s*):(?<name>.+),(\s*):(?<type>\w+),(\s*)(?<position>\d+)(, \"(?<enum_class>.*?)\")?/
+    # Example 
+    # optional :father, :message, 10, "Unicorn"
+    # repeated :favourite_numbers, :int32, 5
+    begin 
+      class_name = ""
+      type = ""
+      field_name = ""
+      File.readlines(filename).each do |line|
+        if ! (line =~ regex_class_name).nil? 
+          class_name = $1
+          @metainfo_messageclasses[class_name] = {}
+          @metainfo_enumclasses[class_name] = {}
+        end
+        if ! (line =~ regex_pbdefs).nil?
+          field_name = $1
+          type = $2
+          field_class_name = $4
+          if type == "message"
+            @metainfo_messageclasses[class_name][field_name] = field_class_name
+          elsif type == "enum"
+            @metainfo_enumclasses[class_name][field_name] = field_class_name
+          end
+        end
+      end
+    rescue Exception => e
+      @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
+      raise e
+    end
+    if class_name.nil?
+      @logger.warn("Error 4: class name not found in file  " + filename)
+      raise ArgumentError, "Invalid protobuf file: " + filename
+    end    
   end
 
-  def require_with_metadata_analysis(filename)
-    require filename
+  def pb2_metadata_analyis(filename)
     regex_class_name = /\s*class\s*(?<name>.+?)\s+/
     regex_module_name = /\s*module\s*(?<name>.+?)\s+/
     regex_pbdefs = /\s*(optional|repeated)(\s*):(?<type>.+),(\s*):(?<name>\w+),(\s*)(?<position>\d+)/
@@ -179,37 +333,40 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
         end
         if ! (line =~ regex_class_name).nil? && !classname_found # because it might be declared twice in the file
           class_name << $1
-          @pb_metainfo[class_name] = {}
+          @metainfo_messageclasses[class_name] = {}
           classname_found = true
         end
         if ! (line =~ regex_pbdefs).nil?
           type = $1
           field_name = $2
           if type =~ /::/
-            @pb_metainfo[class_name][field_name] = type.gsub!(/^:/,"")
+            @metainfo_messageclasses[class_name][field_name] = type.gsub!(/^:/,"")
             
           end
         end
       end
     rescue Exception => e
-      @logger.warn("error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
+      @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
+      raise e
     end
     if class_name.nil?
-      @logger.warn("error 4: class name not found in file  " + filename)
+      @logger.warn("Error 4: class name not found in file  " + filename)
+      raise ArgumentError, "Invalid protobuf file: " + filename
     end    
   end
 
-  def require_pb_path(dir_or_file)
-    f = dir_or_file.end_with? ('.rb')
+  def load_protobuf_definition(filename)
     begin
-      if f
-        @logger.debug("Including protobuf file: " + dir_or_file)
-        require_with_metadata_analysis dir_or_file
+      if filename.end_with? ('.rb')
+        @logger.debug("Including protobuf file: " + filename)
+        require filename
+        if @protobuf_version_3
+          pb3_metadata_analyis(filename)
+        else
+          pb2_metadata_analyis(filename)
+        end
       else 
-        Dir[ dir_or_file + '/*.rb'].each { |file|
-          @logger.debug("Including protobuf path: " + dir_or_file + "/" + file)
-          require_with_metadata_analysis file 
-        }
+        @logger.warn("Not a ruby file: " + filename)
       end
     end
   end

data/logstash-codec-protobuf.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '1.0.3'
+  s.version         = '1.0.5'
   s.licenses        = ['Apache License (2.0)']
-  s.summary         = "This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages"
+  s.summary         = "Reads protobuf messages and converts to Logstash Events"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Inga Feick"]
   s.email           = 'inga.feick@trivago.com'
@@ -20,7 +20,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'ruby-protocol-buffers' # used by the compiled version of our protobuf definition. 
+  s.add_runtime_dependency 'google-protobuf', '3.1'
+  s.add_runtime_dependency 'ruby-protocol-buffers' # for protobuf 2
   s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/codecs/protobuf3_spec.rb

@@ -0,0 +1,147 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/protobuf"
+require "logstash/event"
+require "insist"
+
+
+require 'google/protobuf' # for protobuf3
+
+
+describe LogStash::Codecs::Protobuf do
+
+
+  context "#decodePB3" do
+
+
+    #### Test case 1: Decode simple protobuf ####################################################################################################################
+    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => ['spec/helpers/pb3/unicorn_pb.rb'], "protobuf_version_3" => true)  }
+    before do
+        plugin_unicorn.register      
+    end
+
+    it "should return an event from protobuf encoded data" do
+    
+      unicorn_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
+      data = {:name => 'Pinkie', :age => 18, :is_pegasus => false, :favourite_numbers => [4711,23], :fur_colour => Colour::PINK, 
+      :favourite_colours => [Colour::GREEN, Colour::BLUE]
+      }
+      
+      unicorn_object = unicorn_class.new(data)
+      bin = unicorn_class.encode(unicorn_object)
+      plugin_unicorn.decode(bin) do |event|
+        expect(event.get("name") ).to eq(data[:name] )
+        expect(event.get("age") ).to eq(data[:age])
+        expect(event.get("fur_colour") ).to eq("PINK")
+        expect(event.get("favourite_numbers") ).to eq(data[:favourite_numbers])
+        expect(event.get("favourite_colours") ).to eq(["GREEN","BLUE"])
+        expect(event.get("is_pegasus") ).to eq(data[:is_pegasus] )
+      end
+    end # it
+
+
+
+
+
+    #### Test case 2: decode nested protobuf ####################################################################################################################
+    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => ['spec/helpers/pb3/unicorn_pb.rb'], "protobuf_version_3" => true)  }
+    before do
+        plugin_unicorn.register      
+    end
+
+    it "should return an event from protobuf encoded data with nested classes" do
+    
+
+      unicorn_class = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
+
+      father = unicorn_class.new({:name=> "Sparkle", :age => 50, :fur_colour => 3 })
+      data = {:name => 'Glitter', :fur_colour => Colour::GLITTER, :father => father}   
+     
+      unicorn_object = unicorn_class.new(data)
+      bin = unicorn_class.encode(unicorn_object)
+      plugin_unicorn.decode(bin) do |event|
+        expect(event.get("name") ).to eq(data[:name] )
+        expect(event.get("fur_colour") ).to eq("GLITTER" )
+        expect(event.get("father")["name"] ).to eq(data[:father][:name] )
+        expect(event.get("father")["age"] ).to eq(data[:father][:age] )
+        expect(event.get("father")["fur_colour"] ).to eq("SILVER")
+
+      end
+    end # it
+
+
+  end # context #decodePB3
+
+
+  context "#encodePB3-a" do
+
+    #### Test case 3: encode simple protobuf ####################################################################################################################
+
+    definitions_file = 'spec/helpers/pb3/unicorn_pb.rb'
+    require definitions_file
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [definitions_file], "protobuf_version_3" => true)
+    end
+
+    event3 = LogStash::Event.new("name" => "Pinkie", "age" => 18, "is_pegasus" => false, "favourite_numbers" => [1,2,3], "fur_colour" => Colour::PINK, "favourite_colours" => [1,5]      )
+
+    it "should return protobuf encoded data for testcase 3" do
+
+      subject.on_event do |event, data|
+        insist { data.is_a? String }
+
+        pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
+        decoded_data = pb_builder.decode(data) 
+        expect(decoded_data.name ).to eq(event.get("name") )
+        expect(decoded_data.age ).to eq(event.get("age") )
+        expect(decoded_data.is_pegasus ).to eq(event.get("is_pegasus") )
+        expect(decoded_data.fur_colour ).to eq(:PINK)
+        expect(decoded_data.favourite_numbers ).to eq(event.get("favourite_numbers") )
+        expect(decoded_data.favourite_colours ).to eq([:BLUE,:WHITE] )
+      end # subject.on_event
+      subject.encode(event3)
+    end # it
+
+  end # context
+
+  context "#encodePB3-b" do
+
+    #### Test case 4: encode nested protobuf ####################################################################################################################
+
+    definitions_file = 'spec/helpers/pb3/unicorn_pb.rb'
+    require definitions_file
+
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "Unicorn", "include_path" => [definitions_file], "protobuf_version_3" => true)
+    end
+
+    event4 = LogStash::Event.new("name" => "Horst", "age" => 23, "is_pegasus" => true, "mother" => \
+        {"name" => "Mom", "age" => 47}, "father" => {"name"=> "Daddy", "age"=> 50, "fur_colour" => 3 } # 3 == SILVER      
+      )
+
+    it "should return protobuf encoded data for testcase 4" do
+
+      subject.on_event do |event, data|
+        insist { data.is_a? String }
+
+        pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
+        decoded_data = pb_builder.decode(data) 
+        
+        expect(decoded_data.name ).to eq(event.get("name") )
+        expect(decoded_data.age ).to eq(event.get("age") )
+        expect(decoded_data.is_pegasus ).to eq(event.get("is_pegasus") )
+        expect(decoded_data.mother.name ).to eq(event.get("mother")["name"] )
+        expect(decoded_data.mother.age ).to eq(event.get("mother")["age"] )
+        expect(decoded_data.father.name ).to eq(event.get("father")["name"] )
+        expect(decoded_data.father.age ).to eq(event.get("father")["age"] )
+        expect(decoded_data.father.fur_colour ).to eq(:SILVER)
+
+      
+      end # subject4.on_event
+      subject.encode(event4)
+    end # it
+
+  end # context #encodePB3
+
+end # describe

data/spec/codecs/protobuf_spec.rb

@@ -2,17 +2,21 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/protobuf"
 require "logstash/event"
-require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers
 require "insist"
 
+
+require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers, for protobuf2
+
+
 describe LogStash::Codecs::Protobuf do
 
 
-  context "#decode" do
+
+  context "#decodePB2" do
 
 
     #### Test case 1: Decode simple protobuf bytes for unicorn ####################################################################################################################
-    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Unicorn", "include_path" => ['spec/helpers/unicorn.pb.rb'])  }
+    let(:plugin_unicorn) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Unicorn", "include_path" => ['spec/helpers/pb2/unicorn.pb.rb'])  }
     before do
         plugin_unicorn.register      
     end
@@ -37,7 +41,7 @@ describe LogStash::Codecs::Protobuf do
 
   
  
-    let(:plugin_human) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => ['spec/helpers/human.pb.rb'])  }
+    let(:plugin_human) { LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => ['spec/helpers/pb2/human.pb.rb'])  }
     before do
         plugin_human.register      
     end
@@ -79,7 +83,7 @@ describe LogStash::Codecs::Protobuf do
 
   
  
-    let(:plugin_col) { LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/ColourTestcase.pb.rb'])  }
+    let(:plugin_col) { LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/pb2/ColourTestcase.pb.rb'])  }
     before do
         plugin_col.register      
     end
@@ -98,7 +102,7 @@ describe LogStash::Codecs::Protobuf do
     end # it
 
 
-  end # context
+  end # context decodePB2
 
 
 
@@ -106,9 +110,9 @@ describe LogStash::Codecs::Protobuf do
 
   #### Test case 4: Encode simple protobuf bytes for unicorn ####################################################################################################################
 
-  context "#encode" do
+  context "#encodePB2-a" do
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::UnicornEvent", "include_path" => ['spec/helpers/unicorn_event.pb.rb']) 
+      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::UnicornEvent", "include_path" => ['spec/helpers/pb2/unicorn_event.pb.rb']) 
     end
 
     event = LogStash::Event.new("colour" => "pink", "horn_length" => 12, "last_seen" => 1410081999, "has_wings" => true)    
@@ -134,9 +138,9 @@ describe LogStash::Codecs::Protobuf do
   #### Test case 5: encode complex protobuf bytes for human #####################################################################################################################
   
   
-  context "#encode2" do
+  context "#encodePB2-b" do
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => ['spec/helpers/human.pb.rb']) 
+      next LogStash::Codecs::Protobuf.new("class_name" => "Animal::Human", "include_path" => ['spec/helpers/pb2/human.pb.rb']) 
     end
 
     event = LogStash::Event.new("first_name" => "Jimmy", "middle_names" => ["Bob", "James"], "last_name" => "Doe" \
@@ -174,12 +178,12 @@ describe LogStash::Codecs::Protobuf do
   
 
  
-  context "#encode3" do
+  context "#encodePB2-c" do
     subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/ColourTestcase.pb.rb'])
+      next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/pb2/ColourTestcase.pb.rb'])
     end
 
-    require 'spec/helpers/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
+    require 'spec/helpers/pb2/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
     event = LogStash::Event.new("booleantest" =>  [false, false, true], "least_liked" => ColourProtoTest::Colour::YELLOW,  "favourite_colours" => \
        [ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )    
 
@@ -202,4 +206,4 @@ describe LogStash::Codecs::Protobuf do
 
 
 
-end
+end # describe

data/spec/helpers/pb3/unicorn.proto3

@@ -0,0 +1,31 @@
+syntax = "proto3";
+
+// Compile: protoc --ruby_out=. spec/helpers/*proto3
+
+message Unicorn {
+
+    string name = 1;
+    uint32 age = 2;
+    Colour fur_colour = 3;
+    repeated Colour favourite_colours = 4;
+    repeated int32 favourite_numbers = 5;
+    bool is_pegasus = 6;
+    string timestamp = 7; # Logstash adds this to the event, needed for encoder test
+    string version = 8; # Logstash adds this to the event, needed for encoder test
+    Unicorn mother = 9;
+    Unicorn father = 10;
+}
+
+enum Colour {
+    
+    BLUE = 1;
+    PINK = 2;
+    SILVER = 3;
+    GLITTER = 4;
+    WHITE = 5;
+    GREEN = 6;
+}
+
+// TODO test float datatypes aswell as single fields
+// TODO test class reference to other file as single field
+// TODO test a class as list

data/spec/helpers/pb3/unicorn_pb.rb

@@ -0,0 +1,31 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: spec/helpers/unicorn.proto3
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "Unicorn" do
+    optional :name, :string, 1
+    optional :age, :uint32, 2
+    optional :fur_colour, :enum, 3, "Colour"
+    repeated :favourite_colours, :enum, 4, "Colour"
+    repeated :favourite_numbers, :int32, 5
+    optional :is_pegasus, :bool, 6
+    optional :timestamp, :string, 7
+    optional :version, :string, 8
+    optional :mother, :message, 9, "Unicorn"
+    optional :father, :message, 10, "Unicorn"
+  end
+  add_enum "Colour" do
+    value :UNDEFINED, 0
+    value :BLUE, 1
+    value :PINK, 2
+    value :SILVER, 3
+    value :GLITTER, 4
+    value :WHITE, 5
+    value :GREEN, 6
+  end
+end
+
+Unicorn = Google::Protobuf::DescriptorPool.generated_pool.lookup("Unicorn").msgclass
+Colour = Google::Protobuf::DescriptorPool.generated_pool.lookup("Colour").enummodule

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-protobuf
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.5
 platform: ruby
 authors:
 - Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2018-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
@@ -24,16 +24,30 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  name: google-protobuf
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   name: ruby-protocol-buffers
@@ -41,13 +55,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -55,7 +69,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -73,13 +87,30 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/codecs/protobuf.rb
+- lib/net/jpountz/lz4/lz4/1.3.0/lz4-1.3.0.jar
+- lib/org/apache/kafka/kafka-clients/0.11.0.0/kafka-clients-0.11.0.0.jar
+- lib/org/apache/logging/log4j/log4j-api/2.8.2/log4j-api-2.8.2.jar
+- lib/org/apache/logging/log4j/log4j-slf4j-impl/2.8.2/log4j-slf4j-impl-2.8.2.jar
+- lib/org/slf4j/slf4j-api/1.7.24/slf4j-api-1.7.24.jar
+- lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
+- lib/org/xerial/snappy/snappy-java/1.1.2.6/snappy-java-1.1.2.6.jar
 - logstash-codec-protobuf.gemspec
+- spec/codecs/protobuf3_spec.rb
 - spec/codecs/protobuf_spec.rb
-- spec/helpers/ColourTestcase.pb.rb
-- spec/helpers/ColourTestcase.proto
-- spec/helpers/human.pb.rb
-- spec/helpers/unicorn.pb.rb
-- spec/helpers/unicorn_event.pb.rb
+- spec/helpers/pb2/ColourTestcase.pb.rb
+- spec/helpers/pb2/ColourTestcase.proto
+- spec/helpers/pb2/human.pb.rb
+- spec/helpers/pb2/unicorn.pb.rb
+- spec/helpers/pb2/unicorn_event.pb.rb
+- spec/helpers/pb3/unicorn.proto3
+- spec/helpers/pb3/unicorn_pb.rb
+- vendor/jar-dependencies/runtime-jars/kafka-clients-0.11.0.0.jar
+- vendor/jar-dependencies/runtime-jars/log4j-api-2.8.2.jar
+- vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.8.2.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.3.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.24.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.25.jar
+- vendor/jar-dependencies/runtime-jars/snappy-java-1.1.2.6.jar
 homepage:
 licenses:
 - Apache License (2.0)
@@ -92,24 +123,27 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
-summary: This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages
+summary: Reads protobuf messages and converts to Logstash Events
 test_files:
+- spec/codecs/protobuf3_spec.rb
 - spec/codecs/protobuf_spec.rb
-- spec/helpers/ColourTestcase.pb.rb
-- spec/helpers/ColourTestcase.proto
-- spec/helpers/human.pb.rb
-- spec/helpers/unicorn.pb.rb
-- spec/helpers/unicorn_event.pb.rb
+- spec/helpers/pb2/ColourTestcase.pb.rb
+- spec/helpers/pb2/ColourTestcase.proto
+- spec/helpers/pb2/human.pb.rb
+- spec/helpers/pb2/unicorn.pb.rb
+- spec/helpers/pb2/unicorn_event.pb.rb
+- spec/helpers/pb3/unicorn.proto3
+- spec/helpers/pb3/unicorn_pb.rb