logstash-codec-protobuf 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: c7759d64f37dcbb075892f198b46795d1b35964b66341e70107d13a9065de70b
-  data.tar.gz: db8bfbab2b9cdd7f3dbdbdaa8c0f8b12ab46b7ae4ef024cbc0b35706f3dc08e5
+SHA1:
+  metadata.gz: 4812c43d851064eacffc5c6fba4719eed4691702
+  data.tar.gz: 88436b13daa031ffb714a0f3b1c9611e4130133a
 SHA512:
-  metadata.gz: 72147c0788aed161306b74f8537259b04c615313c85d803e51eca96abdfd0bfdee2fb0f415fbd2eb56e66f08e69a46e8e43403175966ec9178dcfa0133e90e4c
-  data.tar.gz: 1872037c0dff0b05cfaba1d9ac6fe663b12d8130cdab8931d5a3e8b4fc03af9b48720120835bf5acd941a1c0814276a797f9f7a1081ee0d5e4bdfdc63d444fab
+  metadata.gz: b9e6b75105a06e434fdf6f968e80842ac874bbeb7cf1bd72b7339ce3434e43dd4cdc20e992add4b24e811302f2d4366ee8548426b8813042649ea73617cd14ca
+  data.tar.gz: 53d72f5bdb6b5eac6fb88b4d092dba366c820043a709a0d433cf0ef30e7a52f468a48e46cd8a9e728a2fb9c2775c31fee305fb11e5e4f1d7ac0f7d491389a4dc

data/README.md

@@ -27,6 +27,9 @@ Here's an example for a kafka input with protobuf 2:
 	{
 	  zk_connect => "127.0.0.1"
 	  topic_id => "unicorns_protobuffed"
+	  key_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+	  value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+
 	  codec => protobuf 
 	  {
 	    class_name => "Animals::Unicorn"
@@ -40,13 +43,23 @@ Example for protobuf 3:
 	{
 	  zk_connect => "127.0.0.1"
 	  topic_id => "unicorns_protobuffed"
+	  key_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+	  value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
 	  codec => protobuf 
 	  {
-      class_name => "Animals.Unicorn"
+	    class_name => "Animals.Unicorn"
 	    include_path => ['/path/to/pb_definitions/Animal_pb.rb', '/path/to/pb_definitions/Unicorn_pb.rb']
 	    protobuf_version => 3
 	  }
-	}	 
+	}
+
+For version 3 class names check the bottom of the generated protobuf ruby file. It contains lines like this:
+
+    Animals.Unicorn = Google::Protobuf::DescriptorPool.generated_pool.lookup("Animals.Unicorn").msgclass
+
+Use the parameter for the lookup call as the class_name for the codec config.
+
+If you're using a kafka input please also set the deserializer classes as shown above.
 
 ### Class loading order
 

data/docs/index.asciidoc

@@ -30,6 +30,8 @@ kafka
 {
  zk_connect => "127.0.0.1"
  topic_id => "your_topic_goes_here"
+ key_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+ value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
  codec => protobuf 
  {
    class_name => "Animal::Unicorn"
@@ -37,6 +39,7 @@ kafka
  }
 }
 
+Specifically for the kafka input: please set the deserializer classes as shown above.
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Protobuf Codec Configuration Options
@@ -70,6 +73,11 @@ module Foods
            # here are your field definitions.
 
 If your class references other definitions: you only have to add the main class here.
+For version 3 class names check the bottom of the generated protobuf ruby file. It contains lines like this:
+
+    Animals.Unicorn = Google::Protobuf::DescriptorPool.generated_pool.lookup("Animals.Unicorn").msgclass
+
+Use the parameter for the lookup call as the class_name for the codec config.
 
 [id="plugins-{type}s-{plugin}-include_path"]
 ===== `include_path` 

data/lib/logstash/codecs/protobuf.rb

@@ -4,17 +4,36 @@ require 'logstash/util/charset'
 require 'google/protobuf' # for protobuf3
 require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers, for protobuf2
 
-# This codec converts protobuf encoded messages into logstash events and vice versa. 
+# Monkey-patch the `Google::Protobuf::DescriptorPool` with a mutex for exclusive
+# access.
+#
+# The DescriptorPool instance is not thread-safe when loading protobuf
+# definitions. This can cause unrecoverable errors when registering multiple
+# concurrent pipelines that try to register the same dependency. The
+# DescriptorPool instance is global to the JVM and shared among all pipelines.
+class << Google::Protobuf::DescriptorPool
+  def with_lock
+    if !@mutex
+      @mutex = Mutex.new
+    end
+
+    return @mutex
+  end
+end
+
+# This codec converts protobuf encoded messages into logstash events and vice versa.
 #
 # Requires the protobuf definitions as ruby files. You can create those using the [ruby-protoc compiler](https://github.com/codekitchen/ruby-protocol-buffers).
-# 
+#
 # The following shows a usage example for decoding protobuf 2 encoded events from a kafka stream:
 # [source,ruby]
-# kafka 
+# kafka
 # {
 #  zk_connect => "127.0.0.1"
 #  topic_id => "your_topic_goes_here"
-#  codec => protobuf 
+#  key_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+#  value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+#  codec => protobuf
 #  {
 #    class_name => "Animal::Unicorn"
 #    include_path => ['/path/to/protobuf/definitions/UnicornProtobuf.pb.rb']
@@ -23,11 +42,13 @@ require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffer
 #
 # Same example for protobuf 3:
 # [source,ruby]
-# kafka 
+# kafka
 # {
 #  zk_connect => "127.0.0.1"
 #  topic_id => "your_topic_goes_here"
-#  codec => protobuf 
+#  key_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+#  value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+#  codec => protobuf
 #  {
 #    class_name => "Animal.Unicorn"
 #    include_path => ['/path/to/protobuf/definitions/UnicornProtobuf_pb.rb']
@@ -35,6 +56,7 @@ require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffer
 #  }
 # }
 #
+# Specifically for the kafka input: please set the deserializer classes as shown above.
 
 class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   config_name 'protobuf'
@@ -43,7 +65,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   # If your protobuf 2 definition contains modules, prepend them to the class name with double colons like so:
   # [source,ruby]
   # class_name => "Animal::Horse::Unicorn"
-  # 
+  #
   # This corresponds to a protobuf definition starting as follows:
   # [source,ruby]
   # module Animal
@@ -54,14 +76,44 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   # For protobuf 3 separate the modules with single dots.
   # [source,ruby]
   # class_name => "Animal.Horse.Unicorn"
-  # 
+  # Check the bottom of the generated protobuf ruby file. It contains lines like this:
+  # [source,ruby]
+  # Animals.Unicorn = Google::Protobuf::DescriptorPool.generated_pool.lookup("Animals.Unicorn").msgclass
+  # Use the parameter for the lookup call as the class_name for the codec config.
+  #
   # If your class references other definitions: you only have to add the main class here.
   config :class_name, :validate => :string, :required => true
 
-  # List of absolute pathes to files with protobuf definitions. 
-  # When using more than one file, make sure to arrange the files in reverse order of dependency so that each class is loaded before it is 
+  # Relative path to the ruby file that contains class_name
+  #
+  # Relative path (from `protobuf_root_directory`) that holds the definition of the class specified in
+  # `class_name`.
+  #
+  # `class_file` and `include_path` cannot be used at the same time.
+  config :class_file, :validate => :string, :default => '', :required => false
+
+  # Absolute path to the directory that contains all compiled protobuf files.
+  #
+  # Absolute path to the root directory that contains all referenced/used dependencies
+  # of the main class (`class_name`) or any of its dependencies.
+  #
+  # For instance:
+  #
+  # pb3
+  #   ├── header
+  #   │   └── header_pb.rb
+  #   ├── messageA_pb.rb
+  #
+  # In this case `messageA_pb.rb` has an embedded message from `header/header_pb.rb`.
+  # If `class_file` is set to `messageA_pb.rb`, and `class_name` to
+  # `MessageA`, `protobuf_root_directory` must be set to `/path/to/pb3`. Which includes
+  #  both definitions.
+  config :protobuf_root_directory, :validate => :string, :required => false
+
+  # List of absolute pathes to files with protobuf definitions.
+  # When using more than one file, make sure to arrange the files in reverse order of dependency so that each class is loaded before it is
   # refered to by another.
-  # 
+  #
   # Example: a class _Unicorn_ referencing another protobuf class _Wings_
   # [source,ruby]
   # module Animal
@@ -76,15 +128,16 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   # [source,ruby]
   # include_path => ['/path/to/protobuf/definitions/Wings.pb.rb','/path/to/protobuf/definitions/Unicorn.pb.rb']
   #
-  # When using the codec in an output plugin: 
-  # * make sure to include all the desired fields in the protobuf definition, including timestamp. 
+  # When using the codec in an output plugin:
+  # * make sure to include all the desired fields in the protobuf definition, including timestamp.
   #   Remove fields that are not part of the protobuf definition from the event by using the mutate filter.
-  # * the @ symbol is currently not supported in field names when loading the protobuf definitions for encoding. Make sure to call the timestamp field "timestamp" 
+  # * the @ symbol is currently not supported in field names when loading the protobuf definitions for encoding. Make sure to call the timestamp field "timestamp"
   #   instead of "@timestamp" in the protobuf file. Logstash event fields will be stripped of the leading @ before conversion.
-  #  
-  config :include_path, :validate => :array, :required => true
+  #
+  # `class_file` and `include_path` cannot be used at the same time.
+  config :include_path, :validate => :array, :default => [], :required => false
 
-  # Protocol buffer version switch. Defaults to version 2. Please note that the behaviour for enums varies between the versions. 
+  # Protocol buffer version switch. Defaults to version 2. Please note that the behaviour for enums varies between the versions.
   # For protobuf 2 you will get integer representations for enums, for protobuf 3 you'll get string representations due to a different converter library.
   # Recommendation: use the translate plugin to restore previous behaviour when upgrading.
   config :protobuf_version, :validate => [2,3], :default => 2, :required => true
@@ -92,18 +145,61 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   # To tolerate faulty messages that cannot be decoded, set this to false. Otherwise the pipeline will stop upon encountering a non decipherable message.
   config :stop_on_error, :validate => :boolean, :default => false, :required => false
 
+  attr_reader :execution_context
+
+  # id of the pipeline whose events you want to read from.
+  def pipeline_id
+    respond_to?(:execution_context) && !execution_context.nil? ? execution_context.pipeline_id : "main"
+  end
+
   def register
     @metainfo_messageclasses = {}
     @metainfo_enumclasses = {}
     @metainfo_pb2_enumlist = []
-    include_path.each { |path| load_protobuf_definition(path) }
-    if @protobuf_version == 3   
-      @pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
-    else
-      @pb_builder = pb2_create_instance(class_name)
+
+    if @include_path.length > 0 and not class_file.strip.empty?
+      raise LogStash::ConfigurationError, "Cannot use `include_path` and `class_file` at the same time"
+    end
+
+    if @include_path.length == 0 and class_file.strip.empty?
+      raise LogStash::ConfigurationError, "Need to specify `include_path` or `class_file`"
+    end
+
+    should_register = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).nil?
+
+    unless @protobuf_root_directory.nil? or @protobuf_root_directory.strip.empty?
+      if !$LOAD_PATH.include? @protobuf_root_directory and should_register
+        $LOAD_PATH.unshift(@protobuf_root_directory)
+      end
+    end
+
+    @class_file = "#{@protobuf_root_directory}/#{@class_file}" unless (Pathname.new @class_file).absolute? or @class_file.empty?
+
+    # exclusive access while loading protobuf definitions
+    Google::Protobuf::DescriptorPool.with_lock.synchronize do
+      # load from `class_file`
+      load_protobuf_definition(@class_file) if should_register and !@class_file.empty?
+      # load from `include_path`
+      include_path.each { |path| load_protobuf_definition(path) } if include_path.length > 0 and should_register
+
+      if @protobuf_version == 3
+        @pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
+      else
+        @pb_builder = pb2_create_instance(class_name)
+      end
     end
   end
 
+  # Pipelines using this plugin cannot be reloaded.
+  # https://github.com/elastic/logstash/pull/6499
+  #
+  # The DescriptorPool instance registers the protobuf classes (and
+  # dependencies) as global objects. This makes it very difficult to reload a
+  # pipeline, because `class_name` and all of its dependencies are already
+  # registered.
+  def reloadable?
+    return false
+  end
 
   def decode(data)
     if @protobuf_version == 3
@@ -111,7 +207,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
       h = pb3_deep_to_hash(decoded)
     else
       decoded = @pb_builder.parse(data.to_s)
-      h = decoded.to_hash        
+      h = decoded.to_hash
     end
     yield LogStash::Event.new(h) if block_given?
   rescue => e
@@ -126,7 +222,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     if @protobuf_version == 3
       protobytes = pb3_encode_wrapper(event)
     else
-      protobytes = pb2_encode_wrapper(event)     
+      protobytes = pb2_encode_wrapper(event)
     end
      @on_event.call(event, protobytes)
   end # def encode
@@ -139,7 +235,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
       result = Hash.new
       input.to_hash.each {|key, value|
         result[key] = pb3_deep_to_hash(value) # the key is required for the class lookup of enums.
-      }      
+      }
     when ::Array
       result = []
       input.each {|value|
@@ -174,32 +270,32 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
 
   def pb3_encode(datahash, class_name)
     if datahash.is_a?(::Hash)
-      
-    
+
+
 
       # Preparation: the data cannot be encoded until certain criteria are met:
       # 1) remove @ signs from keys.
       # 2) convert timestamps and other objects to strings
       datahash = datahash.inject({}){|x,(k,v)| x[k.gsub(/@/,'').to_sym] = (should_convert_to_string?(v) ? v.to_s : v); x}
-      
+
       # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
       meta = @metainfo_messageclasses[class_name]
       if meta
         meta.map do | (field_name,class_name) |
           key = field_name.to_sym
           if datahash.include?(key)
-            original_value = datahash[key] 
-            datahash[key] = 
+            original_value = datahash[key]
+            datahash[key] =
               if original_value.is_a?(::Array)
                 # make this field an array/list of protobuf objects
                 # value is a list of hashed complex objects, each of which needs to be protobuffed and
                 # put back into the list.
-                original_value.map { |x| pb3_encode(x, class_name) } 
+                original_value.map { |x| pb3_encode(x, class_name) }
                 original_value
-              else 
+              else
                 r = pb3_encode(original_value, class_name)
                 builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
-                builder.new(r)              
+                builder.new(r)
               end # if is array
           end # if datahash_include
         end # do
@@ -214,7 +310,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
             original_value = datahash[key]
             datahash[key] = case original_value
             when ::Array
-              original_value.map { |x| pb3_encode(x, class_name) } 
+              original_value.map { |x| pb3_encode(x, class_name) }
               original_value
             when Fixnum
               original_value # integers will be automatically converted into enum
@@ -229,8 +325,8 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
               # rescue => e
               #   @logger.debug("Encoding error 3: could not translate #{original_value} into enum. #{e}")
               #   raise e
-              # end         
-            end 
+              # end
+            end
           end # if datahash_include
         end # do
       end # if meta
@@ -252,27 +348,27 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
 
 
 
-  def pb2_encode(datahash, class_name)    
+  def pb2_encode(datahash, class_name)
     if datahash.is_a?(::Hash)
       # Preparation: the data cannot be encoded until certain criteria are met:
       # 1) remove @ signs from keys.
       # 2) convert timestamps and other objects to strings
       datahash = ::Hash[datahash.map{|(k,v)| [k.to_s.dup.gsub(/@/,''), (should_convert_to_string?(v) ? v.to_s : v)] }]
-      
+
       # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
       meta = @metainfo_messageclasses[class_name]
       if meta
         meta.map do | (k,c) |
           if datahash.include?(k)
-            original_value = datahash[k]            
-            datahash[k] = 
+            original_value = datahash[k]
+            datahash[k] =
               if original_value.is_a?(::Array)
                 # make this field an array/list of protobuf objects
                 # value is a list of hashed complex objects, each of which needs to be protobuffed and
                 # put back into the list.
-                original_value.map { |x| pb2_encode(x, c) } 
+                original_value.map { |x| pb2_encode(x, c) }
                 original_value
-              else 
+              else
                 proto_obj = pb2_create_instance(c)
                 proto_obj.new(pb2_encode(original_value, c)) # this line is reached in the colourtest for an enum. Enums should not be instantiated. Should enums even be in the messageclasses? I dont think so! TODO bug
               end # if is array
@@ -288,7 +384,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     !(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
   end
 
-  
+
   def pb2_create_instance(name)
     @logger.debug("Creating instance of " + name)
     name.split('::').inject(Object) { |n,c| n.const_get c }
@@ -302,7 +398,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     type = ""
     field_name = ""
     File.readlines(filename).each do |line|
-      if ! (line =~ regex_class_name).nil? 
+      if ! (line =~ regex_class_name).nil?
         class_name = $1
         @metainfo_messageclasses[class_name] = {}
         @metainfo_enumclasses[class_name] = {}
@@ -326,7 +422,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
     raise e
   end
-      
+
 
 
   def pb2_metadata_analyis(filename)
@@ -359,13 +455,13 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
         if type =~ /::/
           clean_type = type.gsub(/^:/,"")
           e = @metainfo_pb2_enumlist.include? clean_type.downcase
-          
+
           if e
             if not @metainfo_enumclasses.key? class_name
               @metainfo_enumclasses[class_name] = {}
             end
             @metainfo_enumclasses[class_name][field_name] = clean_type
-          else            
+          else
             if not @metainfo_messageclasses.key? class_name
               @metainfo_messageclasses[class_name] = {}
             end
@@ -381,32 +477,32 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   rescue LoadError => e
     raise ArgumentError.new("Could not load file: " + filename + ". Please try to use absolute pathes. Current working dir: " + Dir.pwd + ", loadpath: " + $LOAD_PATH.join(" "))
   rescue => e
-    
+
     @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
     raise e
   end
- 
+
 
   def load_protobuf_definition(filename)
     if filename.end_with? ('.rb')
+      # Add to the loading path of the protobuf definitions
       if (Pathname.new filename).absolute?
-        require filename
-      else
-        require_relative filename # needed for the test cases
-        r = File.expand_path(File.dirname(__FILE__))
-        filename = File.join(r, filename) # make the path absolute 
+        begin
+          require filename
+        rescue Exception => e
+          @logger.error("Unable to load file: #{filename}. Reason: #{e.inspect}")
+        end
       end
-     
+
       if @protobuf_version == 3
         pb3_metadata_analyis(filename)
       else
         pb2_metadata_analyis(filename)
       end
-      
-    else 
+
+    else
       @logger.warn("Not a ruby file: " + filename)
     end
   end
 
-
 end # class LogStash::Codecs::Protobuf