logstash-codec-protobuf 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 32ac341961786a035ed34e5e221de5e6944c2226
-  data.tar.gz: 8bb727065267603b4d0d771e2ab1937e7eb83660
+  metadata.gz: 0571d20de470f854fce5c65a479811846d0c3223
+  data.tar.gz: 4d0447fadbd0d520ce33b299253712c7beae65d4
 SHA512:
-  metadata.gz: ba83ef5a43fa82e25d6cb248fa8076982b43d833df1d2ee62ee69c045bedd2664ff04a5b112d880344ebaf470116a176ab008bbccad9f6aab368c30b513ce5d1
-  data.tar.gz: 9699d09931ca8bac5d0242c4952b8496467d8c586032decdce6213ae956e43053b08039115a3deee7db7eaa74fd2d8a9700c32a12d794dcbaf338cbb01419c82
+  metadata.gz: defc91159982c93264d62fddfdbd4677dac242e2b6a49e310275c5cb7095e2b2be25f88cfd5e40f27381bce100711bab21ec9b96a76da6f5a7c838d481a438ee
+  data.tar.gz: 0923ffd9b10199f952ca12dad21e040a0b049b10129dd9332a3078a7269f73e9d8e7b163aaa7c8d205802ffe0c91aee861a47ea411c8d418e3cb8732cbf3f356

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## 0.1.3
+ - Changes for compatibility with logstash 5: remove mechanism to guarantee string based hash keys.
+
+## 0.1.2
+ - First version of this plugin

data/DEVELOPER.md

@@ -1,2 +1,2 @@
-# logstash-codec-example
-Example codec plugin. This should help bootstrap your effort to write your own codec plugin!
+# logstash-codec-protobuf
+No special instructions!

data/LICENSE

@@ -1,13 +1,203 @@
-Copyright (c) 2012–2015 Trivago (http://www.trivago.com/)
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright (c) 2012–2016 Elasticsearch http://www.elastic.co
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

data/NOTICE.TXT

@@ -1,4 +1,4 @@
-Copyright Trivago http://www.trivago.com/
+Copyright (c) 2012-2016 Elasticsearch
 
 This product includes software developed by The Apache Software
 Foundation (http://www.apache.org/).

data/README.md

@@ -74,3 +74,4 @@ Maybe your protobuf definition does not fullfill the requirements and needs addi
 
 * maybe add support for setting undefined fields from default values in the decoder
 
+

data/lib/logstash/codecs/protobuf.rb

@@ -1,164 +1,187 @@
 # encoding: utf-8
 require 'logstash/codecs/base'
 require 'logstash/util/charset'
-require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers
-
+require 'protocol_buffers' 
+
+# This codec converts protobuf encoded messages into logstash events and vice versa. 
+#
+# Requires the protobuf definitions as ruby files. You can create those using the [ruby-protoc compiler](https://github.com/codekitchen/ruby-protocol-buffers).
+# 
+# The following shows a usage example for decoding events from a kafka stream:
+# [source,ruby]
+# kafka 
+# {
+#  zk_connect => "127.0.0.1"
+#  topic_id => "your_topic_goes_here"
+#  codec => protobuf 
+#  {
+#    class_name => "Animal::Unicorn"
+#    include_path => ['/path/to/protobuf/definitions/UnicornProtobuf.pb.rb']
+#  }
+# }
+#
 class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   config_name 'protobuf'
 
-  # Required: list of strings containing directories or files with protobuf definitions
-  config :include_path, :validate => :array, :required => true
-
-  # Name of the class to decode
+  # Name of the class to decode.
+  # If your protobuf definition contains modules, prepend them to the class name with double colons like so:
+  # [source,ruby]
+  # class_name => "Foods::Dairy::Cheese"
+  # 
+  # This corresponds to a protobuf definition starting as follows:
+  # [source,ruby]
+  # module Foods
+  #    module Dairy
+  #        class Cheese
+  #            # here are your field definitions.
+  # 
+  # If your class references other definitions: you only have to add the main class here.
   config :class_name, :validate => :string, :required => true
 
-  # For benchmarking only, not intended for public use: change encoder strategy. 
-  # valid method names are:  encoder_strategy_1 (the others are not implemented yet)
-  config :encoder_method, :validate => :string, :default => "encoder_strategy_1"
+  # List of absolute pathes to files with protobuf definitions. 
+  # When using more than one file, make sure to arrange the files in reverse order of dependency so that each class is loaded before it is 
+  # refered to by another.
+  # 
+  # Example: a class _Cheese_ referencing another protobuf class _Milk_
+  # [source,ruby]
+  # module Foods
+  #   module Dairy
+  #         class Cheese
+  #            set_fully_qualified_name "Foods.Dairy.Cheese"
+  #            optional ::Foods::Cheese::Milk, :milk, 1
+  #            optional :int64, :unique_id, 2
+  #            # here be more field definitions
+  #
+  # would be configured as
+  # [source,ruby]
+  # include_path => ['/path/to/protobuf/definitions/Milk.pb.rb','/path/to/protobuf/definitions/Cheese.pb.rb']
+  #
+  # When using the codec in an output plugin: 
+  # * make sure to include all the desired fields in the protobuf definition, including timestamp. 
+  #   Remove fields that are not part of the protobuf definition from the event by using the mutate filter.
+  # * the @ symbol is currently not supported in field names when loading the protobuf definitions for encoding. Make sure to call the timestamp field "timestamp" 
+  #   instead of "@timestamp" in the protobuf file. Logstash event fields will be stripped of the leading @ before conversion.
+  #  
+  config :include_path, :validate => :array, :required => true
 
   def register
-    @pb_metainfo = {}
-    include_path.each { |path| require_pb_path(path) }
-    @obj = create_object_from_name(class_name)
-    @logger.debug("Protobuf files successfully loaded.")
-
+    @pb_class_references = {}
+    include_path.each { |path| load_protobuf_classfiles(path) }
+    @protobuf_class = create_protobuf_object(@class_name)
+    
   end
 
   def decode(data)
-    decoded = @obj.parse(data.to_s)
-    results = keys2strings(decoded.to_hash)
-    yield LogStash::Event.new(results) if block_given?
+    decoded = @protobuf_class.parse(data.to_s)
+    yield LogStash::Event.new(decoded.to_hash) if block_given?
   end # def decode
 
-  def keys2strings(data)
-    if data.is_a?(::Hash)
-      new_hash = Hash.new
-      data.each{|k,v| new_hash[k.to_s] = keys2strings(v)}
-      new_hash
-    else
-      data
-    end
-  end
-
 
   def encode(event)
-    protobytes = generate_protobuf(event)
-    @on_event.call(event, protobytes)
-  end # def encode
-
-  private
-  def generate_protobuf(event)  
-    meth = self.method(encoder_method)
-    data = meth.call(event, @class_name)
     begin
-      msg = @obj.new(data)
-      msg.serialize_to_string
+      data = prepare_nested_objects(event.to_hash, @class_name)
+      pbo = @protobuf_class.new(data) 
+      protobytes = pbo.serialize_to_string
+      @on_event.call(event, protobytes)
     rescue NoMethodError
-      @logger.debug("error 2: NoMethodError. Maybe mismatching protobuf definition. Required fields are: " + event.to_hash.keys.join(", "))
+      @logger.warn("Error 2: NoMethodError. Maybe mismatching protobuf definition? Make sure that your protobuf definition has at least these fields: " + event.to_hash.keys.join(", "))
+    rescue => e
+      @logger.warn("Could not encode protobuf: " + e.message)
     end
-  end
-
-  def encoder_strategy_1(event, class_name)
-    _encoder_strategy_1(event.to_hash, class_name)
-
-  end
+  end # def encode
 
-  def _encoder_strategy_1(datahash, class_name)
-    fields = clean_hash_keys(datahash)
-    fields = flatten_hash_values(fields) # TODO we could merge this and the above method back into one to save one iteration, but how are we going to name it?
-    meta = get_complex_types(class_name) # returns a hash with member names and their protobuf class names
-    meta.map do | (k,typeinfo) |
+  # Creates instances of nested protobuf references recursively. TODO improve documentation
+  private
+  def prepare_nested_objects(fields, class_name)
+    fields = prepare_for_encoding(fields)
+    referenced_classes = @pb_class_references[class_name] # returns a hash with member names and their protobuf class names
+    referenced_classes.map do | (k,class_name) |
       if fields.include?(k)
-        original_value = fields[k] 
-        proto_obj = create_object_from_name(typeinfo)
+        value = fields[k] 
+        proto_obj = create_protobuf_object(class_name)
         fields[k] = 
-          if original_value.is_a?(::Array) 
-            ecs1_list_helper(original_value, proto_obj, typeinfo)
-            
+          if value.is_a?(::Array) 
+            # make this field an array/list of protobuf objects
+            # value is a list of hashed complex objects, each of which needs to be protobuffed and
+            # put back into the list.
+            value.map { |x| prepare_nested_objects(x, class_name) } 
+            value
           else 
-            recursive_fix = _encoder_strategy_1(original_value, class_name)
-            proto_obj.new(recursive_fix)
+            proto_obj.new( prepare_nested_objects(value, class_name) )
           end # if is array
       end
-
     end 
-    
     fields
   end
 
-  def ecs1_list_helper(value, proto_obj, class_name)
-    # make this field an array/list of protobuf objects
-    # value is a list of hashed complex objects, each of which needs to be protobuffed and
-    # put back into the list.
-    next unless value.is_a?(::Array)
-    value.map { |x| _encoder_strategy_1(x, class_name) } 
-    value
-  end
 
-  def flatten_hash_values(datahash)
-    # 2) convert timestamps and other objects to strings
-    next unless datahash.is_a?(::Hash)
-    
-    ::Hash[datahash.map{|(k,v)| [k, (convert_to_string?(v) ? v.to_s : v)] }]
-  end
 
-  def clean_hash_keys(datahash)
-    # 1) remove @ signs from keys 
+  # Removes @ characters from the member names of the event.
+  # Necessary for @timestamp fields and the likes. Otherwise we'd run into errors (no such method) upon creating the protobuf object.
+  # Then convert timestamps and other objects to strings so that they can be passed to the protobuf object constructor method.
+  def prepare_for_encoding(datahash)
     next unless datahash.is_a?(::Hash)
-    
-    ::Hash[datahash.map{|(k,v)| [remove_atchar(k.to_s), v] }]
-  end #clean_hash_keys
+    ::Hash[datahash.map{|(k,v)| [k.to_s.gsub(/@/,'').to_sym, convert_value(v)] }]
+  end
 
-  def convert_to_string?(v)
-    !(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
+  def convert_value(v)
+    (convertable_to_string?(v) ? v.to_s : v)    
   end
 
-   
-  def remove_atchar(key) # necessary for @timestamp fields and the likes. Protobuf definition doesn't handle @ in field names well.
-    key.dup.gsub(/@/,'')
+  def convertable_to_string?(v)
+    !(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
   end
 
-  private
-  def create_object_from_name(name)
+  # Creates an instance of a protobuf class name. This instance will be used later to call the decode and encode methods on.
+  def create_protobuf_object(name)
     begin
-      @logger.debug("Creating instance of " + name)
+      
       name.split('::').inject(Object) { |n,c| n.const_get c }
      end
   end
 
-  def get_complex_types(class_name)
-    @pb_metainfo[class_name]
-  end
 
-  def require_with_metadata_analysis(filename)
-    require filename
+
+  # Analyses a protobuf definition on which other protobuf classes it uses. 
+  # This is needed for the encoder section of the codec. 
+  # When encoding an event into a pb class which uses other pb classes, we need to create the 
+  # objects for those nested classes first, so that we can reference them when encoding the topmost 
+  # class. In order to be able to do so, this method reads each protobuf class line by line and
+  # stores the information in the @pb_class_references member.
+  # Params:
+  # +filename+:: the absolute path to the protobuf definition.
+  def load_class_reference_information(filename)
     regex_class_name = /\s*class\s*(?<name>.+?)\s+/
     regex_module_name = /\s*module\s*(?<name>.+?)\s+/
     regex_pbdefs = /\s*(optional|repeated)(\s*):(?<type>.+),(\s*):(?<name>\w+),(\s*)(?<position>\d+)/
-    # now we also need to find out which class it contains and the protobuf definitions in it.
-    # We'll unfortunately need that later so that we can create nested objects.
     begin 
       class_name = ""
       type = ""
       field_name = ""
       classname_found = false
       File.readlines(filename).each do |line|
-        if ! (line =~ regex_module_name).nil? && !classname_found # because it might be declared twice in the file
+        # Check if the current line contains the module name (but only if the class name hasn't been found yet because it might be declared twice in the file)
+        if ! (line =~ regex_module_name).nil? && !classname_found 
+          # Module name found, so we start to create the class name string which starts with the module.
           class_name << $1 
           class_name << "::"
-    
         end
-        if ! (line =~ regex_class_name).nil? && !classname_found # because it might be declared twice in the file
+        
+        # Check if the current line contains the class name (but only if it hasn't been found yet because it might be declared twice in the file)
+        if ! (line =~ regex_class_name).nil? && !classname_found
+          # class name found. Let's append it to the class name string, which might already contain the module name
           class_name << $1
-          @pb_metainfo[class_name] = {}
+          # initialize the hash for the field specific information that we will collect in the next step
+          @pb_class_references[class_name] = {}
           classname_found = true
         end
+
         if ! (line =~ regex_pbdefs).nil?
           type = $1
           field_name = $2
           if type =~ /::/
-            @pb_metainfo[class_name][field_name] = type.gsub!(/^:/,"")
-            
+            # the line contains a field declaration which references another class. We need to store the name of that class.
+            @pb_class_references[class_name][field_name] = type.gsub!(/^:/,"")
           end
         end
       end
@@ -170,16 +193,21 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     end    
   end
 
-  def require_pb_path(dir_or_file)
-    f = dir_or_file.end_with? ('.rb')
+  # This method calls 'require' for the protobuf class files listed in the 'include_path' section of the config.
+  # When given a directory instead of a file, it will require all files in the directory.
+  # Params:
+  # +dir_or_file+:: the absolute path to the file or directory that need to be loaded
+  def load_protobuf_classfiles(dir_or_file)
     begin
-      if f
-        @logger.debug("Including protobuf file: " + dir_or_file)
-        require_with_metadata_analysis dir_or_file
+      if dir_or_file.end_with? ('.rb')
+        
+        require dir_or_file
+        load_class_reference_information dir_or_file
       else 
         Dir[ dir_or_file + '/*.rb'].each { |file|
-          @logger.debug("Including protobuf path: " + dir_or_file + "/" + file)
-          require_with_metadata_analysis file 
+          
+          require file
+          load_class_reference_information file 
         }
       end
     end

data/logstash-codec-protobuf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '0.1.2'
+  s.version         = '0.1.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,8 +19,9 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 3.0.0'
-  s.add_runtime_dependency 'ruby-protocol-buffers' # used by the compiled version of our protobuf definition. 
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'ruby-protocol-buffers' # https://github.com/codekitchen/ruby-protocol-buffers
+  # s.add_runtime_dependency 'google-protobuf' # https://github.com/google/protobuf/tree/master/ruby
   s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/codecs/protobuf_spec.rb

@@ -23,10 +23,11 @@ describe LogStash::Codecs::Protobuf do
       unicorn = Animal::Unicorn.new(data)
         
       plugin_unicorn.decode(unicorn.serialize_to_string) do |event|
-        expect(event["colour"] ).to eq(data[:colour] )
-        expect(event["horn_length"] ).to eq(data[:horn_length] )
-        expect(event["last_seen"] ).to eq(data[:last_seen] )
-        expect(event["has_wings"] ).to eq(data[:has_wings] )
+        expect(event.get("colour") ).to eq(data[:colour] )
+        expect(event.get("horn_length") ).to eq(data[:horn_length] )
+        expect(event.get("last_seen") ).to eq(data[:last_seen] )
+        expect(event.get("has_wings") ).to eq(data[:has_wings] )
+       
       end
     end # it
 
@@ -54,18 +55,20 @@ describe LogStash::Codecs::Protobuf do
       hugo = Animal::Human.new(data)
        
       plugin_human.decode(hugo.serialize_to_string) do |event|
-        expect(event["first_name"] ).to eq(data[:first_name] )
-        expect(event["middle_names"] ).to eq(data[:middle_names] )
-        expect(event["last_name"] ).to eq(data[:last_name] )
-        expect(event["mother"]["first_name"] ).to eq(data_m[:first_name] ) 
-        expect(event["father"]["first_name"] ).to eq(data_f[:first_name] )
-        expect(event["mother"]["last_name"] ).to eq(data_m[:last_name] )
-        expect(event["mother"]["mother"]["last_name"] ).to eq(data_gm[:last_name] )
-        expect(event["mother"]["mother"]["first_name"] ).to eq(data_gm[:first_name] )
-        expect(event["mother"]["mother"]["middle_names"] ).to eq(data_gm[:middle_names] )
-        expect(event["mother"]["mother"]["vegetarian"] ).to eq(data_gm[:vegetarian] )
-        expect(event["father"]["last_name"] ).to eq(data_f[:last_name] )
-        expect(event["father"]["middle_names"] ).to eq(data_f[:middle_names] )
+        expect(event.get("first_name") ).to eq(data[:first_name] )
+        expect(event.get("middle_names") ).to eq(data[:middle_names] )
+        expect(event.get("middle_names").length ).to eq(data[:middle_names].length )
+        expect(event.get("[middle_names]")[1] ).to eq(data[:middle_names][1] )
+        expect(event.get("last_name") ).to eq(data[:last_name] )
+        expect(event.get("[mother][first_name]") ).to eq(data_m[:first_name] ) 
+        expect(event.get("[father][first_name]") ).to eq(data_f[:first_name] )
+        expect(event.get("[mother][last_name]") ).to eq(data_m[:last_name] )
+        expect(event.get("[mother][mother][last_name]") ).to eq(data_gm[:last_name] )
+        expect(event.get("[mother][mother][first_name]") ).to eq(data_gm[:first_name] )
+        expect(event.get("[mother][mother][middle_names]") ).to eq(data_gm[:middle_names] )
+        expect(event.get("[mother][mother][vegetarian]") ).to eq(data_gm[:vegetarian] )
+        expect(event.get("[father][last_name]") ).to eq(data_f[:last_name] )
+        expect(event.get("[father][middle_names]") ).to eq(data_f[:middle_names] )
       end
     end # it
 
@@ -91,9 +94,9 @@ describe LogStash::Codecs::Protobuf do
       pb = ColourProtoTest.new(data)
        
       plugin_col.decode(pb.serialize_to_string) do |event|
-        expect(event["least_liked"] ).to eq(data[:least_liked] )
-        expect(event["favourite_colours"] ).to eq(data[:favourite_colours] )
-        expect(event["booleantest"] ).to eq(data[:booleantest] )
+        expect(event.get("least_liked") ).to eq(data[:least_liked] )
+        expect(event.get("favourite_colours") ).to eq(data[:favourite_colours] )
+        expect(event.get("booleantest") ).to eq(data[:booleantest] )
       end
     end # it
 
@@ -118,10 +121,10 @@ describe LogStash::Codecs::Protobuf do
         insist { data.is_a? String }
         unicorn = Animal::UnicornEvent.parse(data) 
     
-        expect(unicorn.colour ).to eq(event["colour"] )
-        expect(unicorn.horn_length ).to eq(event["horn_length"] )
-        expect(unicorn.last_seen ).to eq(event["last_seen"] )
-        expect(unicorn.has_wings ).to eq(event["has_wings"] )
+        expect(unicorn.colour ).to eq(event.get("colour") )
+        expect(unicorn.horn_length ).to eq(event.get("horn_length") )
+        expect(unicorn.last_seen ).to eq(event.get("last_seen") )
+        expect(unicorn.has_wings ).to eq(event.get("has_wings") )
       
       end # subject.on_event
       subject.encode(event)
@@ -149,17 +152,17 @@ describe LogStash::Codecs::Protobuf do
         insist { data.is_a? String }
         jimmy = Animal::Human.parse(data) 
         
-        expect(jimmy.first_name ).to eq(event["first_name"] )
-        expect(jimmy.middle_names ).to eq(event["middle_names"] )
-        expect(jimmy.last_name ).to eq(event["last_name"] )
-        expect(jimmy.mother.first_name ).to eq(event["mother"]["first_name"] )
-        expect(jimmy.father.first_name ).to eq(event["father"]["first_name"] )
-        expect(jimmy.mother.middle_names ).to eq(event["mother"]["middle_names"] )
-        expect(jimmy.mother.age ).to eq(event["mother"]["age"] ) # recursion test for values
-        expect(jimmy.mother.vegetarian ).to eq(event["mother"]["vegetarian"] ) # recursion test for values
-        expect(jimmy.father.last_name ).to eq(event["father"]["last_name"] )
-        expect(jimmy.father.email ).to eq(event["father"]["@email"] ) # recursion test for keys
-        expect(jimmy.mother.last_name ).to eq(event["mother"]["last_name"] )
+        expect(jimmy.first_name ).to eq(event.get("first_name") )
+        expect(jimmy.middle_names ).to eq(event.get("middle_names") )
+        expect(jimmy.last_name ).to eq(event.get("last_name") )
+        expect(jimmy.mother.first_name ).to eq(event.get("[mother][first_name]") )
+        expect(jimmy.father.first_name ).to eq(event.get("[father][first_name]") )
+        expect(jimmy.mother.middle_names ).to eq(event.get("[mother][middle_names]") )
+        expect(jimmy.mother.age ).to eq(event.get("[mother][age]") ) # recursion test for values
+        expect(jimmy.mother.vegetarian ).to eq(event.get("[mother][vegetarian]") ) # recursion test for values
+        expect(jimmy.father.last_name ).to eq(event.get("[father][last_name]") )
+        expect(jimmy.father.email ).to eq(event.get("[father][@email]") ) # recursion test for keys
+        expect(jimmy.mother.last_name ).to eq(event.get("[mother][last_name]") )
       
       end # subject.on_event
       subject.encode(event)
@@ -190,9 +193,9 @@ describe LogStash::Codecs::Protobuf do
 
         colpref = ColourProtoTest.parse(data) 
         
-        expect(colpref.booleantest ).to eq(event["booleantest"] )
-        expect(colpref.least_liked ).to eq(event["least_liked"] )
-        expect(colpref.favourite_colours ).to eq(event["favourite_colours"] )
+        expect(colpref.booleantest ).to eq(event.get("booleantest") )
+        expect(colpref.least_liked ).to eq(event.get("least_liked") )
+        expect(colpref.favourite_colours ).to eq(event.get("favourite_colours") )
 
       
       end # subject.on_event
@@ -202,4 +205,30 @@ describe LogStash::Codecs::Protobuf do
 
 
 
+  context "#encode4" do
+    subject do
+      next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/ColourTestcase.pb.rb'])
+    end
+
+    require 'spec/helpers/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
+    event = LogStash::Event.new("booleantest" =>  [false, false, true], "least_liked" => ColourProtoTest::Colour::YELLOW,  "favourite_colours" => \
+       [ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )    
+
+    it "should return protobuf encoded data from a complex event with enums" do
+
+      subject.on_event do |event, data|
+        insist { data.is_a? String }
+
+        colpref = ColourProtoTest.parse(data) 
+        
+        expect(colpref.booleantest ).to eq(event.get("booleantest") )
+        expect(colpref.least_liked ).to eq(event.get("least_liked") )
+        expect(colpref.favourite_colours ).to eq(event.get("favourite_colours") )
+
+      
+      end # subject.on_event
+      subject.encode(event)
+    end # it
+  end # context
+
 end

data/spec/helpers/unicorn.pb.rb

@@ -14,6 +14,7 @@ module Animal
     optional :int32, :horn_length, 2
     optional :int32, :last_seen, 3
     optional :bool, :has_wings, 4
+    optional :float, :height, 5
   end
 
 end

metadata

@@ -1,63 +1,63 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-22 00:00:00.000000000 Z
+date: 2016-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+        version: '1.60'
+    - - <=
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: ruby-protocol-buffers
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.60'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: ruby-protocol-buffers
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: inga.feick@trivago.com
 executables: []