logstash-codec-protobuf 1.0.5 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a623045ad79469eeefd501735a5fc3284c19eef3
-  data.tar.gz: 1a4b69f9ae131f5abe2b55270073c975edcbeab3
+SHA256:
+  metadata.gz: c7759d64f37dcbb075892f198b46795d1b35964b66341e70107d13a9065de70b
+  data.tar.gz: db8bfbab2b9cdd7f3dbdbdaa8c0f8b12ab46b7ae4ef024cbc0b35706f3dc08e5
 SHA512:
-  metadata.gz: 0e152a7b9675dbd1c430da257a340364944f71b430bd093913705f376971650cc46933395a492f38732dc3665472ca56ebdc3e995d0c5023236b67466a7dc835
-  data.tar.gz: a55114ffbc512e1ee31748093f864c1683e33536f590ded503cbae5ca2ec4f61754e7d518efc722593c9af4040270438f40fa05d3fa03ae27e334d1a61ff520c
+  metadata.gz: 72147c0788aed161306b74f8537259b04c615313c85d803e51eca96abdfd0bfdee2fb0f415fbd2eb56e66f08e69a46e8e43403175966ec9178dcfa0133e90e4c
+  data.tar.gz: 1872037c0dff0b05cfaba1d9ac6fe663b12d8130cdab8931d5a3e8b4fc03af9b48720120835bf5acd941a1c0814276a797f9f7a1081ee0d5e4bdfdc63d444fab

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-## 1.0.5
+## 1.1.0
   - Add support for protobuf3
 
 ## 1.0.4

data/README.md

@@ -4,51 +4,74 @@ This is a codec plugin for [Logstash](https://github.com/elastic/logstash) to pa
 
 # Prerequisites and Installation
  
-* prepare your ruby versions of the protobuf definitions, for example using the ruby-protoc compiler from https://github.com/codekitchen/ruby-protocol-buffers
+* prepare your ruby versions of the protobuf definitions
+** For protobuf 2 use the [ruby-protoc compiler](https://github.com/codekitchen/ruby-protocol-buffers).
+** For protobuf 3 use the [official google protobuf compiler](https://developers.google.com/protocol-buffers/docs/reference/ruby-generated).
 * install the codec: `bin/logstash-plugin install logstash-codec-protobuf`
 * use the codec in your logstash config file. See details below.
 
 ## Configuration
 
-include_path  (required): an array of strings with filenames or directory names where logstash can find your protobuf definitions. Please provide absolute paths. For directories it will only try to import files ending on .rb
+include_path  (required): an array of strings with filenames where logstash can find your protobuf definitions. Please provide absolute paths. For directories it will only try to import files ending on .rb
 
-class_name    (required): the name of the protobuf class that is to be decoded or encoded.
+class_name    (required): the name of the protobuf class that is to be decoded or encoded. For protobuf 2 separate the modules with ::. For protobuf 3 use single dots. See examples below.
+
+protobuf_version (optional): set this to 3 if you want to use protobuf 3 definitions. Defaults to 2.
 
 ## Usage example: decoder
 
 Use this as a codec in any logstash input. Just provide the name of the class that your incoming objects will be encoded in, and specify the path to the compiled definition.
-Here's an example for a kafka input:
+Here's an example for a kafka input with protobuf 2:
+
+	kafka 
+	{
+	  zk_connect => "127.0.0.1"
+	  topic_id => "unicorns_protobuffed"
+	  codec => protobuf 
+	  {
+	    class_name => "Animals::Unicorn"
+	    include_path => ['/path/to/pb_definitions/Animal.pb.rb', '/path/to/pb_definitions/Unicorn.pb.rb']
+	  }
+	}
+
+Example for protobuf 3:
 
 	kafka 
-  	{
-      zk_connect => "127.0.0.1"
-      topic_id => "unicorns_protobuffed"
-      codec => protobuf 
-      {
-        class_name => "Unicorn"
-        include_path => ['/my/path/to/compiled/protobuf/definitions/UnicornProtobuf.pb.rb']
-      }
-  	}   
+	{
+	  zk_connect => "127.0.0.1"
+	  topic_id => "unicorns_protobuffed"
+	  codec => protobuf 
+	  {
+      class_name => "Animals.Unicorn"
+	    include_path => ['/path/to/pb_definitions/Animal_pb.rb', '/path/to/pb_definitions/Unicorn_pb.rb']
+	    protobuf_version => 3
+	  }
+	}	 
 
-### Example with referenced definitions
+### Class loading order
 
-Imagine you have the following protobuf relationship: class Cheese lives in namespace Foods::Dairy and uses another class Milk. 
+Imagine you have the following protobuf version 2 relationship: class Unicorn lives in namespace Animal::Horse and uses another class Wings. 
 
-	module Foods
-  		module Dairy
-    		class Cheese
-    			set_fully_qualified_name "Foods.Dairy.Cheese"
-			    optional ::Foods::Cheese::Milk, :milk, 1
-			    optional :int64, :unique_id, 2
-			    # here be more field definitions
+	module Animal
+	  module Horse
+	    class Unicorn
+	      set_fully_qualified_name "Animal.Horse.Unicorn"
+	      optional ::Animal::Bodypart::Wings, :wings, 1
+	      optional :string, :name, 2
+	      # here be more field definitions
 
-Make sure to put the referenced Milk class first in the include_path:
+Make sure to put the referenced wings class first in the include_path:
 
-	include_path => ['/path/to/protobuf/definitions/Milk.pb.rb','/path/to/protobuf/definitions/Cheese.pb.rb']
+	include_path => ['/path/to/pb_definitions/wings.pb.rb','/path/to/pb_definitions/unicorn.pb.rb']
 
 Set the class name to the parent class:
 	
-	class_name => "Foods::Dairy::Cheese"
+	class_name => "Animal::Horse::Unicorn"
+
+for protobuf 2. For protobuf 3 use 
+
+	class_name => "Animal.Horse.Unicorn"
+
 
 ## Usage example: encoder
 
@@ -59,14 +82,18 @@ The configuration of the codec for encoding logstash events for a protobuf outpu
 
 ## Troubleshooting
 
-### "uninitialized constant SOME_CLASS_NAME"
+### Protobuf 2 
+#### "uninitialized constant SOME_CLASS_NAME"
 
 If you include more than one definition class, consider the order of inclusion. This is especially relevant if you include whole directories. A definition might refer to another definition that is not loaded yet. In this case, please specify the files in the include_path variable in reverse order of reference. See 'Example with referenced definitions' above.
 
-### no protobuf output
+#### no protobuf output
+
+Maybe your protobuf definition does not fullfill the requirements and needs additional fields. Run logstash with the --debug flag and search for error messages.
 
-Maybe your protobuf definition does not fullfill the requirements and needs additional fields. Run logstash with the --debug flag and grep for "error 2".
+### Protobuf 3
 
+Tba.
 
 ## Limitations and roadmap
 

data/lib/logstash/codecs/protobuf.rb

@@ -8,7 +8,7 @@ require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffer
 #
 # Requires the protobuf definitions as ruby files. You can create those using the [ruby-protoc compiler](https://github.com/codekitchen/ruby-protocol-buffers).
 # 
-# The following shows a usage example for decoding events from a kafka stream:
+# The following shows a usage example for decoding protobuf 2 encoded events from a kafka stream:
 # [source,ruby]
 # kafka 
 # {
@@ -21,21 +21,39 @@ require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffer
 #  }
 # }
 #
+# Same example for protobuf 3:
+# [source,ruby]
+# kafka 
+# {
+#  zk_connect => "127.0.0.1"
+#  topic_id => "your_topic_goes_here"
+#  codec => protobuf 
+#  {
+#    class_name => "Animal.Unicorn"
+#    include_path => ['/path/to/protobuf/definitions/UnicornProtobuf_pb.rb']
+#    protobuf_version => 3
+#  }
+# }
+#
 
 class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   config_name 'protobuf'
 
   # Name of the class to decode.
-  # If your protobuf definition contains modules, prepend them to the class name with double colons like so:
+  # If your protobuf 2 definition contains modules, prepend them to the class name with double colons like so:
   # [source,ruby]
-  # class_name => "Foods::Dairy::Cheese"
+  # class_name => "Animal::Horse::Unicorn"
   # 
   # This corresponds to a protobuf definition starting as follows:
   # [source,ruby]
-  # module Foods
-  #    module Dairy
-  #        class Cheese
-  #            # here are your field definitions.
+  # module Animal
+  #   module Horse
+  #     class Unicorn
+  #       # here are your field definitions.
+  #
+  # For protobuf 3 separate the modules with single dots.
+  # [source,ruby]
+  # class_name => "Animal.Horse.Unicorn"
   # 
   # If your class references other definitions: you only have to add the main class here.
   config :class_name, :validate => :string, :required => true
@@ -44,19 +62,19 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   # When using more than one file, make sure to arrange the files in reverse order of dependency so that each class is loaded before it is 
   # refered to by another.
   # 
-  # Example: a class _Cheese_ referencing another protobuf class _Milk_
+  # Example: a class _Unicorn_ referencing another protobuf class _Wings_
   # [source,ruby]
-  # module Foods
-  #   module Dairy
-  #         class Cheese
-  #            set_fully_qualified_name "Foods.Dairy.Cheese"
-  #            optional ::Foods::Cheese::Milk, :milk, 1
-  #            optional :int64, :unique_id, 2
-  #            # here be more field definitions
+  # module Animal
+  #   module Horse
+  #     class Unicorn
+  #       set_fully_qualified_name "Animal.Horse.Unicorn"
+  #       optional ::Animal::Bodypart::Wings, :wings, 1
+  #       optional :string, :name, 2
+  #       # here be more field definitions
   #
   # would be configured as
   # [source,ruby]
-  # include_path => ['/path/to/protobuf/definitions/Milk.pb.rb','/path/to/protobuf/definitions/Cheese.pb.rb']
+  # include_path => ['/path/to/protobuf/definitions/Wings.pb.rb','/path/to/protobuf/definitions/Unicorn.pb.rb']
   #
   # When using the codec in an output plugin: 
   # * make sure to include all the desired fields in the protobuf definition, including timestamp. 
@@ -66,17 +84,20 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   #  
   config :include_path, :validate => :array, :required => true
 
-  # Protocol buffer version switch. Set to false (default) for version 2. Please note that the behaviour for enums varies between the versions. 
+  # Protocol buffer version switch. Defaults to version 2. Please note that the behaviour for enums varies between the versions. 
   # For protobuf 2 you will get integer representations for enums, for protobuf 3 you'll get string representations due to a different converter library.
   # Recommendation: use the translate plugin to restore previous behaviour when upgrading.
-  config :protobuf_version_3, :validate => :boolean, :required => true, :default=>false
+  config :protobuf_version, :validate => [2,3], :default => 2, :required => true
 
+  # To tolerate faulty messages that cannot be decoded, set this to false. Otherwise the pipeline will stop upon encountering a non decipherable message.
+  config :stop_on_error, :validate => :boolean, :default => false, :required => false
 
   def register
     @metainfo_messageclasses = {}
     @metainfo_enumclasses = {}
+    @metainfo_pb2_enumlist = []
     include_path.each { |path| load_protobuf_definition(path) }
-    if @protobuf_version_3      
+    if @protobuf_version == 3   
       @pb_builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
     else
       @pb_builder = pb2_create_instance(class_name)
@@ -85,24 +106,24 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
 
 
   def decode(data)
-    begin
-      if @protobuf_version_3
-        decoded = @pb_builder.decode(data.to_s)
-        h = pb3_deep_to_hash(decoded)
-      else
-        decoded = @pb_builder.parse(data.to_s)
-        h = decoded.to_hash        
-      end
-      yield LogStash::Event.new(h) if block_given?
-    rescue => e
-      @logger.warn("Couldn't decode protobuf: #{e.inspect}.")
+    if @protobuf_version == 3
+      decoded = @pb_builder.decode(data.to_s)
+      h = pb3_deep_to_hash(decoded)
+    else
+      decoded = @pb_builder.parse(data.to_s)
+      h = decoded.to_hash        
+    end
+    yield LogStash::Event.new(h) if block_given?
+  rescue => e
+    @logger.warn("Couldn't decode protobuf: #{e.inspect}.")
+    if stop_on_error
       raise e
     end
   end # def decode
 
 
   def encode(event)
-    if @protobuf_version_3
+    if @protobuf_version == 3
       protobytes = pb3_encode_wrapper(event)
     else
       protobytes = pb2_encode_wrapper(event)     
@@ -113,22 +134,23 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
 
   private
   def pb3_deep_to_hash(input)
-    if input.class.ancestors.include? Google::Protobuf::MessageExts # it's a protobuf class
+    case input
+    when Google::Protobuf::MessageExts # it's a protobuf class
       result = Hash.new
       input.to_hash.each {|key, value|
         result[key] = pb3_deep_to_hash(value) # the key is required for the class lookup of enums.
       }      
-    elsif input.kind_of?(Array)
+    when ::Array
       result = []
       input.each {|value|
           result << pb3_deep_to_hash(value)
       }
-    elsif input.kind_of?(::Hash)
+    when ::Hash
       result = {}
       input.each {|key, value|
           result[key] = pb3_deep_to_hash(value)
       }
-    elsif input.instance_of? Symbol # is an Enum
+    when Symbol # is an Enum
       result = input.to_s.sub(':','')
     else
       result = input
@@ -137,65 +159,66 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   end
 
   def pb3_encode_wrapper(event)
-    begin
-      data = pb3_encode(event.to_hash, @class_name)
-      pb_obj = @pb_builder.new(data)
-      @pb_builder.encode(pb_obj)
-    rescue ArgumentError => e
-      @logger.debug("Encoding error 2. Probably mismatching protobuf definition. Required fields in the protobuf definition are: " + event.to_hash.keys.join(", ") + " and the timestamp field name must not include a @. ")
-      raise e
-    rescue => e
-      @logger.debug("Couldn't generate protobuf: ${e}")
-      raise e
-    end
+    data = pb3_encode(event.to_hash, @class_name)
+    pb_obj = @pb_builder.new(data)
+    @pb_builder.encode(pb_obj)
+  rescue ArgumentError => e
+    k = event.to_hash.keys.join(", ")
+    @logger.debug("Encoding error 2. Probably mismatching protobuf definition. Required fields in the protobuf definition are: #{k} and the timestamp field name must not include an @.")
+    raise e
+  rescue => e
+    @logger.debug("Couldn't generate protobuf: #{e.inspect}")
+    raise e
   end
 
 
   def pb3_encode(datahash, class_name)
-    next unless datahash.is_a?(::Hash)
-
-    # Preparation: the data cannot be encoded until certain criteria are met:
-    # 1) remove @ signs from keys.
-    # 2) convert timestamps and other objects to strings
-    datahash = datahash.inject({}){|x,(k,v)| x[k.gsub(/@/,'').to_sym] = (should_convert_to_string?(v) ? v.to_s : v); x}
+    if datahash.is_a?(::Hash)
+      
     
-    # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
-    meta = @metainfo_messageclasses[class_name]
-    if meta
-      meta.map do | (field_name,class_name) |
-        key = field_name.to_sym
-        if datahash.include?(key)
-          original_value = datahash[key] 
-          datahash[key] = 
-            if original_value.is_a?(::Array)
-              # make this field an array/list of protobuf objects
-              # value is a list of hashed complex objects, each of which needs to be protobuffed and
-              # put back into the list.
+
+      # Preparation: the data cannot be encoded until certain criteria are met:
+      # 1) remove @ signs from keys.
+      # 2) convert timestamps and other objects to strings
+      datahash = datahash.inject({}){|x,(k,v)| x[k.gsub(/@/,'').to_sym] = (should_convert_to_string?(v) ? v.to_s : v); x}
+      
+      # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
+      meta = @metainfo_messageclasses[class_name]
+      if meta
+        meta.map do | (field_name,class_name) |
+          key = field_name.to_sym
+          if datahash.include?(key)
+            original_value = datahash[key] 
+            datahash[key] = 
+              if original_value.is_a?(::Array)
+                # make this field an array/list of protobuf objects
+                # value is a list of hashed complex objects, each of which needs to be protobuffed and
+                # put back into the list.
+                original_value.map { |x| pb3_encode(x, class_name) } 
+                original_value
+              else 
+                r = pb3_encode(original_value, class_name)
+                builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
+                builder.new(r)              
+              end # if is array
+          end # if datahash_include
+        end # do
+      end # if meta
+
+      # Check if any of the fields in this hash are enum classes and if so, create a builder for them.
+      meta = @metainfo_enumclasses[class_name]
+      if meta
+        meta.map do | (field_name,class_name) |
+          key = field_name.to_sym
+          if datahash.include?(key)
+            original_value = datahash[key]
+            datahash[key] = case original_value
+            when ::Array
               original_value.map { |x| pb3_encode(x, class_name) } 
               original_value
-            else 
-              r = pb3_encode(original_value, class_name)
-              builder = Google::Protobuf::DescriptorPool.generated_pool.lookup(class_name).msgclass
-              builder.new(r)              
-            end # if is array
-        end # if datahash_include
-      end # do
-    end # if meta
-    # Check if any of the fields in this hash are enum classes and if so, create a builder for them.
-    meta = @metainfo_enumclasses[class_name]
-    if meta
-      meta.map do | (field_name,class_name) |
-        key = field_name.to_sym
-        if datahash.include?(key)
-          original_value = datahash[key]
-          datahash[key] = 
-          if original_value.is_a?(::Array)
-            original_value.map { |x| pb3_encode(x, class_name) } 
-            original_value
-          else
-            if original_value.is_a?(Fixnum)
+            when Fixnum
               original_value # integers will be automatically converted into enum
-            else
+            # else
               # feature request: support for providing integers as strings or symbols.
               # not fully tested yet:
               # begin
@@ -204,62 +227,59 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
               #     mod.const_get(class_name)
               #   end # do
               # rescue => e
-              #   @logger.debug("Encoding error 3: could not translate #{original_value} into enum. ${e}")
+              #   @logger.debug("Encoding error 3: could not translate #{original_value} into enum. #{e}")
               #   raise e
               # end         
-            end # if is a fixnum    
-          end # if is array
-        end # if datahash_include
-      end # do
-    end # if meta
+            end 
+          end # if datahash_include
+        end # do
+      end # if meta
+    end
     datahash
   end
 
   def pb2_encode_wrapper(event)
-    begin
-      data = pb2_encode(event.to_hash, @class_name)
-      msg = @pb_builder.new(data)
-      msg.serialize_to_string
-    rescue NoMethodError => e
-      @logger.debug("Encoding error 2. Probably mismatching protobuf definition. Required fields in the protobuf definition are: " + event.to_hash.keys.join(", ") + " and the timestamp field name must not include a @. ")
-      raise e
-    rescue => e
-      @logger.debug("Encoding error 1: ${e}")
-      raise e
-    end
+    data = pb2_encode(event.to_hash, @class_name)
+    msg = @pb_builder.new(data)
+    msg.serialize_to_string
+  rescue NoMethodError => e
+    @logger.debug("Encoding error 2. Probably mismatching protobuf definition. Required fields in the protobuf definition are: " + event.to_hash.keys.join(", ") + " and the timestamp field name must not include a @. ")
+    raise e
+  rescue => e
+    @logger.debug("Encoding error 1: #{e.inspect}")
+    raise e
   end
 
 
-  def pb2_encode(datahash, class_name)
-    next unless datahash.is_a?(::Hash)
-
-    # Preparation: the data cannot be encoded until certain criteria are met:
-    # 1) remove @ signs from keys.
-    # 2) convert timestamps and other objects to strings
-    datahash = ::Hash[datahash.map{|(k,v)| [k.to_s.dup.gsub(/@/,''), (should_convert_to_string?(v) ? v.to_s : v)] }]
-    
-    # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
-    meta = @metainfo_messageclasses[class_name]
-    if meta
-      meta.map do | (k,class_name) |
-        if datahash.include?(k)
-          original_value = datahash[k] 
-          p
-          datahash[k] = 
-            if original_value.is_a?(::Array)
-              # make this field an array/list of protobuf objects
-              # value is a list of hashed complex objects, each of which needs to be protobuffed and
-              # put back into the list.
-              original_value.map { |x| pb2_encode(x, class_name) } 
-              original_value
-            else 
-              proto_obj = pb2_create_instance(class_name)
-              proto_obj.new(pb2_encode(original_value, class_name))
-            end # if is array
-        end # if datahash_include
-      end # do
-    end # if meta
 
+  def pb2_encode(datahash, class_name)    
+    if datahash.is_a?(::Hash)
+      # Preparation: the data cannot be encoded until certain criteria are met:
+      # 1) remove @ signs from keys.
+      # 2) convert timestamps and other objects to strings
+      datahash = ::Hash[datahash.map{|(k,v)| [k.to_s.dup.gsub(/@/,''), (should_convert_to_string?(v) ? v.to_s : v)] }]
+      
+      # Check if any of the fields in this hash are protobuf classes and if so, create a builder for them.
+      meta = @metainfo_messageclasses[class_name]
+      if meta
+        meta.map do | (k,c) |
+          if datahash.include?(k)
+            original_value = datahash[k]            
+            datahash[k] = 
+              if original_value.is_a?(::Array)
+                # make this field an array/list of protobuf objects
+                # value is a list of hashed complex objects, each of which needs to be protobuffed and
+                # put back into the list.
+                original_value.map { |x| pb2_encode(x, c) } 
+                original_value
+              else 
+                proto_obj = pb2_create_instance(c)
+                proto_obj.new(pb2_encode(original_value, c)) # this line is reached in the colourtest for an enum. Enums should not be instantiated. Should enums even be in the messageclasses? I dont think so! TODO bug
+              end # if is array
+          end # if datahash_include
+        end # do
+      end # if meta
+    end
     datahash
   end
 
@@ -270,104 +290,121 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
 
   
   def pb2_create_instance(name)
-    begin
-      @logger.debug("Creating instance of " + name)
-      name.split('::').inject(Object) { |n,c| n.const_get c }
-     end
+    @logger.debug("Creating instance of " + name)
+    name.split('::').inject(Object) { |n,c| n.const_get c }
   end
 
 
   def pb3_metadata_analyis(filename)
     regex_class_name = /\s*add_message "(?<name>.+?)" do\s+/ # TODO optimize both regexes for speed (negative lookahead)
     regex_pbdefs = /\s*(optional|repeated)(\s*):(?<name>.+),(\s*):(?<type>\w+),(\s*)(?<position>\d+)(, \"(?<enum_class>.*?)\")?/
-    # Example 
-    # optional :father, :message, 10, "Unicorn"
-    # repeated :favourite_numbers, :int32, 5
-    begin 
-      class_name = ""
-      type = ""
-      field_name = ""
-      File.readlines(filename).each do |line|
-        if ! (line =~ regex_class_name).nil? 
-          class_name = $1
-          @metainfo_messageclasses[class_name] = {}
-          @metainfo_enumclasses[class_name] = {}
-        end
-        if ! (line =~ regex_pbdefs).nil?
-          field_name = $1
-          type = $2
-          field_class_name = $4
-          if type == "message"
-            @metainfo_messageclasses[class_name][field_name] = field_class_name
-          elsif type == "enum"
-            @metainfo_enumclasses[class_name][field_name] = field_class_name
-          end
+    class_name = ""
+    type = ""
+    field_name = ""
+    File.readlines(filename).each do |line|
+      if ! (line =~ regex_class_name).nil? 
+        class_name = $1
+        @metainfo_messageclasses[class_name] = {}
+        @metainfo_enumclasses[class_name] = {}
+      end # if
+      if ! (line =~ regex_pbdefs).nil?
+        field_name = $1
+        type = $2
+        field_class_name = $4
+        if type == "message"
+          @metainfo_messageclasses[class_name][field_name] = field_class_name
+        elsif type == "enum"
+          @metainfo_enumclasses[class_name][field_name] = field_class_name
         end
-      end
-    rescue Exception => e
-      @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
-      raise e
-    end
+      end # if
+    end # readlines
     if class_name.nil?
       @logger.warn("Error 4: class name not found in file  " + filename)
       raise ArgumentError, "Invalid protobuf file: " + filename
-    end    
+    end
+  rescue Exception => e
+    @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
+    raise e
   end
+      
+
 
   def pb2_metadata_analyis(filename)
-    regex_class_name = /\s*class\s*(?<name>.+?)\s+/
-    regex_module_name = /\s*module\s*(?<name>.+?)\s+/
+    regex_class_start = /\s*set_fully_qualified_name \"(?<name>.+)\".*?/
+    regex_enum_name = /\s*include ..ProtocolBuffers..Enum\s*/
     regex_pbdefs = /\s*(optional|repeated)(\s*):(?<type>.+),(\s*):(?<name>\w+),(\s*)(?<position>\d+)/
     # now we also need to find out which class it contains and the protobuf definitions in it.
     # We'll unfortunately need that later so that we can create nested objects.
-    begin 
-      class_name = ""
-      type = ""
-      field_name = ""
-      classname_found = false
-      File.readlines(filename).each do |line|
-        if ! (line =~ regex_module_name).nil? && !classname_found # because it might be declared twice in the file
-          class_name << $1 
-          class_name << "::"
-    
-        end
-        if ! (line =~ regex_class_name).nil? && !classname_found # because it might be declared twice in the file
-          class_name << $1
-          @metainfo_messageclasses[class_name] = {}
-          classname_found = true
+
+    class_name = ""
+    type = ""
+    field_name = ""
+    is_enum_class = false
+
+    File.readlines(filename).each do |line|
+      if ! (line =~ regex_enum_name).nil?
+        is_enum_class= true
+       end
+
+      if ! (line =~ regex_class_start).nil?
+        class_name = $1.gsub('.',"::").split('::').map {|word| word.capitalize}.join('::')
+        if is_enum_class
+          @metainfo_pb2_enumlist << class_name.downcase
         end
-        if ! (line =~ regex_pbdefs).nil?
-          type = $1
-          field_name = $2
-          if type =~ /::/
-            @metainfo_messageclasses[class_name][field_name] = type.gsub!(/^:/,"")
-            
+        is_enum_class= false # reset when next class starts
+      end
+      if ! (line =~ regex_pbdefs).nil?
+        type = $1
+        field_name = $2
+        if type =~ /::/
+          clean_type = type.gsub(/^:/,"")
+          e = @metainfo_pb2_enumlist.include? clean_type.downcase
+          
+          if e
+            if not @metainfo_enumclasses.key? class_name
+              @metainfo_enumclasses[class_name] = {}
+            end
+            @metainfo_enumclasses[class_name][field_name] = clean_type
+          else            
+            if not @metainfo_messageclasses.key? class_name
+              @metainfo_messageclasses[class_name] = {}
+            end
+            @metainfo_messageclasses[class_name][field_name] = clean_type
           end
         end
       end
-    rescue Exception => e
-      @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
-      raise e
     end
     if class_name.nil?
       @logger.warn("Error 4: class name not found in file  " + filename)
       raise ArgumentError, "Invalid protobuf file: " + filename
-    end    
+    end
+  rescue LoadError => e
+    raise ArgumentError.new("Could not load file: " + filename + ". Please try to use absolute pathes. Current working dir: " + Dir.pwd + ", loadpath: " + $LOAD_PATH.join(" "))
+  rescue => e
+    
+    @logger.warn("Error 3: unable to read pb definition from file  " + filename+ ". Reason: #{e.inspect}. Last settings were: class #{class_name} field #{field_name} type #{type}. Backtrace: " + e.backtrace.inspect.to_s)
+    raise e
   end
+ 
 
   def load_protobuf_definition(filename)
-    begin
-      if filename.end_with? ('.rb')
-        @logger.debug("Including protobuf file: " + filename)
+    if filename.end_with? ('.rb')
+      if (Pathname.new filename).absolute?
         require filename
-        if @protobuf_version_3
-          pb3_metadata_analyis(filename)
-        else
-          pb2_metadata_analyis(filename)
-        end
-      else 
-        @logger.warn("Not a ruby file: " + filename)
+      else
+        require_relative filename # needed for the test cases
+        r = File.expand_path(File.dirname(__FILE__))
+        filename = File.join(r, filename) # make the path absolute 
+      end
+     
+      if @protobuf_version == 3
+        pb3_metadata_analyis(filename)
+      else
+        pb2_metadata_analyis(filename)
       end
+      
+    else 
+      @logger.warn("Not a ruby file: " + filename)
     end
   end