logstash-codec-protobuf 0.1.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0571d20de470f854fce5c65a479811846d0c3223
-  data.tar.gz: 4d0447fadbd0d520ce33b299253712c7beae65d4
+  metadata.gz: 43ef1232f802b173bf7fee33f894c31145e5032e
+  data.tar.gz: f0d955aeee52c1bbb6f72abe016603c6eac8994e
 SHA512:
-  metadata.gz: defc91159982c93264d62fddfdbd4677dac242e2b6a49e310275c5cb7095e2b2be25f88cfd5e40f27381bce100711bab21ec9b96a76da6f5a7c838d481a438ee
-  data.tar.gz: 0923ffd9b10199f952ca12dad21e040a0b049b10129dd9332a3078a7269f73e9d8e7b163aaa7c8d205802ffe0c91aee861a47ea411c8d418e3cb8732cbf3f356
+  metadata.gz: fb9a3a3b646935acd74d986e3defd67a79ccb14bad91bbd9abe4240bf938ef001a709a8ec05efa243410c25d08e3577a1c3a53e1614bccda55d0a6f07837e719
+  data.tar.gz: ad7d0690b3169ab22623db7b5e8dfd5a17d2cb1e082d54c39da86fef8239dd3576a1d1ed5894dcd27f3fe332f1538d8a71a960cb41a7ee85ab7f5f05c32dc4ef

data/CHANGELOG.md

@@ -1,5 +1,5 @@
-## 0.1.3
- - Changes for compatibility with logstash 5: remove mechanism to guarantee string based hash keys.
-
+## 1.0.0
+ - Update to v5.0 API
+
 ## 0.1.2
  - First version of this plugin

data/Gemfile

@@ -1,2 +1,3 @@
 source 'https://rubygems.org'
+gem 'logstash-docgen', :path => "/Users/suyog/ws/elastic/logstash/tools/logstash-docgen"
 gemspec

data/lib/logstash/codecs/protobuf.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 require 'logstash/codecs/base'
 require 'logstash/util/charset'
-require 'protocol_buffers' 
+require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers
 
 # This codec converts protobuf encoded messages into logstash events and vice versa. 
 #
@@ -20,6 +20,7 @@ require 'protocol_buffers'
 #  }
 # }
 #
+
 class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   config_name 'protobuf'
 
@@ -64,124 +65,153 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   #  
   config :include_path, :validate => :array, :required => true
 
+
+
+
+
   def register
-    @pb_class_references = {}
-    include_path.each { |path| load_protobuf_classfiles(path) }
-    @protobuf_class = create_protobuf_object(@class_name)
-    
+    @pb_metainfo = {}
+    include_path.each { |path| require_pb_path(path) }
+    @obj = create_object_from_name(class_name)
+    @logger.debug("Protobuf files successfully loaded.")
+
   end
 
   def decode(data)
-    decoded = @protobuf_class.parse(data.to_s)
-    yield LogStash::Event.new(decoded.to_hash) if block_given?
+    decoded = @obj.parse(data.to_s)
+    results = keys2strings(decoded.to_hash)
+    yield LogStash::Event.new(results) if block_given?
   end # def decode
 
+  def keys2strings(data)
+    if data.is_a?(::Hash)
+      new_hash = Hash.new
+      data.each{|k,v| new_hash[k.to_s] = keys2strings(v)}
+      new_hash
+    else
+      data
+    end
+  end
+
 
   def encode(event)
+    protobytes = generate_protobuf(event)
+    @on_event.call(event, protobytes)
+  end # def encode
+
+  private
+  def generate_protobuf(event)  
+    meth = self.method("encoder_strategy_1")
+    data = meth.call(event, @class_name)
     begin
-      data = prepare_nested_objects(event.to_hash, @class_name)
-      pbo = @protobuf_class.new(data) 
-      protobytes = pbo.serialize_to_string
-      @on_event.call(event, protobytes)
+      msg = @obj.new(data)
+      msg.serialize_to_string
     rescue NoMethodError
-      @logger.warn("Error 2: NoMethodError. Maybe mismatching protobuf definition? Make sure that your protobuf definition has at least these fields: " + event.to_hash.keys.join(", "))
-    rescue => e
-      @logger.warn("Could not encode protobuf: " + e.message)
+      @logger.debug("error 2: NoMethodError. Maybe mismatching protobuf definition. Required fields are: " + event.to_hash.keys.join(", "))
     end
-  end # def encode
+  end
 
-  # Creates instances of nested protobuf references recursively. TODO improve documentation
-  private
-  def prepare_nested_objects(fields, class_name)
-    fields = prepare_for_encoding(fields)
-    referenced_classes = @pb_class_references[class_name] # returns a hash with member names and their protobuf class names
-    referenced_classes.map do | (k,class_name) |
+  def encoder_strategy_1(event, class_name)
+    _encoder_strategy_1(event.to_hash, class_name)
+
+  end
+
+  def _encoder_strategy_1(datahash, class_name)
+    fields = clean_hash_keys(datahash)
+    fields = flatten_hash_values(fields) # TODO we could merge this and the above method back into one to save one iteration, but how are we going to name it?
+    meta = get_complex_types(class_name) # returns a hash with member names and their protobuf class names
+    meta.map do | (k,typeinfo) |
       if fields.include?(k)
-        value = fields[k] 
-        proto_obj = create_protobuf_object(class_name)
+        original_value = fields[k] 
+        proto_obj = create_object_from_name(typeinfo)
         fields[k] = 
-          if value.is_a?(::Array) 
-            # make this field an array/list of protobuf objects
-            # value is a list of hashed complex objects, each of which needs to be protobuffed and
-            # put back into the list.
-            value.map { |x| prepare_nested_objects(x, class_name) } 
-            value
+          if original_value.is_a?(::Array) 
+            ecs1_list_helper(original_value, proto_obj, typeinfo)
+            
           else 
-            proto_obj.new( prepare_nested_objects(value, class_name) )
+            recursive_fix = _encoder_strategy_1(original_value, class_name)
+            proto_obj.new(recursive_fix)
           end # if is array
       end
+
     end 
+    
     fields
   end
 
+  def ecs1_list_helper(value, proto_obj, class_name)
+    # make this field an array/list of protobuf objects
+    # value is a list of hashed complex objects, each of which needs to be protobuffed and
+    # put back into the list.
+    next unless value.is_a?(::Array)
+    value.map { |x| _encoder_strategy_1(x, class_name) } 
+    value
+  end
 
-
-  # Removes @ characters from the member names of the event.
-  # Necessary for @timestamp fields and the likes. Otherwise we'd run into errors (no such method) upon creating the protobuf object.
-  # Then convert timestamps and other objects to strings so that they can be passed to the protobuf object constructor method.
-  def prepare_for_encoding(datahash)
+  def flatten_hash_values(datahash)
+    # 2) convert timestamps and other objects to strings
     next unless datahash.is_a?(::Hash)
-    ::Hash[datahash.map{|(k,v)| [k.to_s.gsub(/@/,'').to_sym, convert_value(v)] }]
+    
+    ::Hash[datahash.map{|(k,v)| [k, (convert_to_string?(v) ? v.to_s : v)] }]
   end
 
-  def convert_value(v)
-    (convertable_to_string?(v) ? v.to_s : v)    
-  end
+  def clean_hash_keys(datahash)
+    # 1) remove @ signs from keys 
+    next unless datahash.is_a?(::Hash)
+    
+    ::Hash[datahash.map{|(k,v)| [remove_atchar(k.to_s), v] }]
+  end #clean_hash_keys
 
-  def convertable_to_string?(v)
+  def convert_to_string?(v)
     !(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
   end
 
-  # Creates an instance of a protobuf class name. This instance will be used later to call the decode and encode methods on.
-  def create_protobuf_object(name)
+   
+  def remove_atchar(key) # necessary for @timestamp fields and the likes. Protobuf definition doesn't handle @ in field names well.
+    key.dup.gsub(/@/,'')
+  end
+
+  private
+  def create_object_from_name(name)
     begin
-      
+      @logger.debug("Creating instance of " + name)
       name.split('::').inject(Object) { |n,c| n.const_get c }
      end
   end
 
+  def get_complex_types(class_name)
+    @pb_metainfo[class_name]
+  end
 
-
-  # Analyses a protobuf definition on which other protobuf classes it uses. 
-  # This is needed for the encoder section of the codec. 
-  # When encoding an event into a pb class which uses other pb classes, we need to create the 
-  # objects for those nested classes first, so that we can reference them when encoding the topmost 
-  # class. In order to be able to do so, this method reads each protobuf class line by line and
-  # stores the information in the @pb_class_references member.
-  # Params:
-  # +filename+:: the absolute path to the protobuf definition.
-  def load_class_reference_information(filename)
+  def require_with_metadata_analysis(filename)
+    require filename
     regex_class_name = /\s*class\s*(?<name>.+?)\s+/
     regex_module_name = /\s*module\s*(?<name>.+?)\s+/
     regex_pbdefs = /\s*(optional|repeated)(\s*):(?<type>.+),(\s*):(?<name>\w+),(\s*)(?<position>\d+)/
+    # now we also need to find out which class it contains and the protobuf definitions in it.
+    # We'll unfortunately need that later so that we can create nested objects.
     begin 
       class_name = ""
       type = ""
       field_name = ""
       classname_found = false
       File.readlines(filename).each do |line|
-        # Check if the current line contains the module name (but only if the class name hasn't been found yet because it might be declared twice in the file)
-        if ! (line =~ regex_module_name).nil? && !classname_found 
-          # Module name found, so we start to create the class name string which starts with the module.
+        if ! (line =~ regex_module_name).nil? && !classname_found # because it might be declared twice in the file
           class_name << $1 
           class_name << "::"
+    
         end
-        
-        # Check if the current line contains the class name (but only if it hasn't been found yet because it might be declared twice in the file)
-        if ! (line =~ regex_class_name).nil? && !classname_found
-          # class name found. Let's append it to the class name string, which might already contain the module name
+        if ! (line =~ regex_class_name).nil? && !classname_found # because it might be declared twice in the file
           class_name << $1
-          # initialize the hash for the field specific information that we will collect in the next step
-          @pb_class_references[class_name] = {}
+          @pb_metainfo[class_name] = {}
           classname_found = true
         end
-
         if ! (line =~ regex_pbdefs).nil?
           type = $1
           field_name = $2
           if type =~ /::/
-            # the line contains a field declaration which references another class. We need to store the name of that class.
-            @pb_class_references[class_name][field_name] = type.gsub!(/^:/,"")
+            @pb_metainfo[class_name][field_name] = type.gsub!(/^:/,"")
+            
           end
         end
       end
@@ -193,21 +223,16 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     end    
   end
 
-  # This method calls 'require' for the protobuf class files listed in the 'include_path' section of the config.
-  # When given a directory instead of a file, it will require all files in the directory.
-  # Params:
-  # +dir_or_file+:: the absolute path to the file or directory that need to be loaded
-  def load_protobuf_classfiles(dir_or_file)
+  def require_pb_path(dir_or_file)
+    f = dir_or_file.end_with? ('.rb')
     begin
-      if dir_or_file.end_with? ('.rb')
-        
-        require dir_or_file
-        load_class_reference_information dir_or_file
+      if f
+        @logger.debug("Including protobuf file: " + dir_or_file)
+        require_with_metadata_analysis dir_or_file
       else 
         Dir[ dir_or_file + '/*.rb'].each { |file|
-          
-          require file
-          load_class_reference_information file 
+          @logger.debug("Including protobuf path: " + dir_or_file + "/" + file)
+          require_with_metadata_analysis file 
         }
       end
     end

data/logstash-codec-protobuf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '0.1.3'
+  s.version         = '1.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,8 +20,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'ruby-protocol-buffers' # https://github.com/codekitchen/ruby-protocol-buffers
-  # s.add_runtime_dependency 'google-protobuf' # https://github.com/google/protobuf/tree/master/ruby
+  s.add_runtime_dependency 'ruby-protocol-buffers' # used by the compiled version of our protobuf definition. 
   s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/codecs/protobuf_spec.rb

@@ -27,7 +27,6 @@ describe LogStash::Codecs::Protobuf do
         expect(event.get("horn_length") ).to eq(data[:horn_length] )
         expect(event.get("last_seen") ).to eq(data[:last_seen] )
         expect(event.get("has_wings") ).to eq(data[:has_wings] )
-       
       end
     end # it
 
@@ -57,8 +56,6 @@ describe LogStash::Codecs::Protobuf do
       plugin_human.decode(hugo.serialize_to_string) do |event|
         expect(event.get("first_name") ).to eq(data[:first_name] )
         expect(event.get("middle_names") ).to eq(data[:middle_names] )
-        expect(event.get("middle_names").length ).to eq(data[:middle_names].length )
-        expect(event.get("[middle_names]")[1] ).to eq(data[:middle_names][1] )
         expect(event.get("last_name") ).to eq(data[:last_name] )
         expect(event.get("[mother][first_name]") ).to eq(data_m[:first_name] ) 
         expect(event.get("[father][first_name]") ).to eq(data_f[:first_name] )
@@ -205,30 +202,4 @@ describe LogStash::Codecs::Protobuf do
 
 
 
-  context "#encode4" do
-    subject do
-      next LogStash::Codecs::Protobuf.new("class_name" => "ColourProtoTest", "include_path" => ['spec/helpers/ColourTestcase.pb.rb'])
-    end
-
-    require 'spec/helpers/ColourTestcase.pb.rb' # otherwise we cant use the colour enums in the next line
-    event = LogStash::Event.new("booleantest" =>  [false, false, true], "least_liked" => ColourProtoTest::Colour::YELLOW,  "favourite_colours" => \
-       [ColourProtoTest::Colour::BLACK, ColourProtoTest::Colour::BLUE] )    
-
-    it "should return protobuf encoded data from a complex event with enums" do
-
-      subject.on_event do |event, data|
-        insist { data.is_a? String }
-
-        colpref = ColourProtoTest.parse(data) 
-        
-        expect(colpref.booleantest ).to eq(event.get("booleantest") )
-        expect(colpref.least_liked ).to eq(event.get("least_liked") )
-        expect(colpref.favourite_colours ).to eq(event.get("favourite_colours") )
-
-      
-      end # subject.on_event
-      subject.encode(event)
-    end # it
-  end # context
-
 end

data/spec/helpers/unicorn.pb.rb

@@ -14,7 +14,6 @@ module Animal
     optional :int32, :horn_length, 2
     optional :int32, :last_seen, 3
     optional :bool, :has_wings, 4
-    optional :float, :height, 5
   end
 
 end

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-28 00:00:00.000000000 Z
+date: 2016-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - <=
+    - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
@@ -24,16 +24,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - <=
+    - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: ruby-protocol-buffers
@@ -41,13 +41,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -55,7 +55,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -91,17 +91,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages