logstash-codec-protobuf 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae5fff64bbdc72cfc6f1dbbeecd0857481bd1a93
-  data.tar.gz: 5ff0987575c066dd83443a3a4fe1eefc9149f9ea
+  metadata.gz: 92231ace59a2a8218764c99dab51abb5636db440
+  data.tar.gz: d8129bf3a5bf757f42bb3c2e95ae8d74455531b2
 SHA512:
-  metadata.gz: 8875e1c022bc8a7155469838c312a0076a296c99d506bb37f18cbe4ea77b080e034b46a76ec6e0084f07f60e5aad88feafe3157b72408c3213746d1a39dac79c
-  data.tar.gz: e3eb525c75ed899a003767855ed03201dcab2338f1fd33d8a6328045be4b8dad776acd422375155ddc7455ef6a3ab8cb710bc3211b1c483def8126bb25545040
+  metadata.gz: 6a7379eeb8a33c4dd722a01e55bd67f5daeb37ba7ad323ebd41707b359f27a585ca5dcaaf720ecb74d6746c50230fda17249ef849ff3f01489714dbd9c286769
+  data.tar.gz: d2175a9b628f60128295e9e4871d895ba8cb65d33202af0a3c66ac63c0aa4540a173b3b7c508f28de8db3d950f4b72604e308d5138530a79393144244c5188a1

data/CHANGELOG.md

@@ -1,4 +0,0 @@
-* 1.1.0
-  - Handle scalar types (string/number) and be more defensive about crashable errors
-* 1.0.1
-  - Handle JSON arrays at source root by emitting multiple events

data/CONTRIBUTORS

@@ -2,14 +2,9 @@ The following is a list of people who have contributed ideas, code, bug
 reports, or in general have helped logstash along its way.
 
 Contributors:
-* Colin Surprenant (colinsurprenant)
-* Jordan Sissel (jordansissel)
-* João Duarte (jsvd)
-* Kurt Hurtado (kurtado)
-* Nick Ethier (nickethier)
-* Pier-Hugues Pellerin (ph)
-* Richard Pijnenburg (electrical)
-* Tal Levy (talevy)
+* Inga Feick (ingafeick)
+* Nicolai Schulten (krakenfuss)
+
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2015 Trivago (http://www.trivago.com/)
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/NOTICE.TXT

@@ -1,5 +1,4 @@
-Elasticsearch
-Copyright 2012-2015 Elasticsearch
+Copyright Trivago http://www.trivago.com/
 
 This product includes software developed by The Apache Software
 Foundation (http://www.apache.org/).

data/README.md

@@ -2,15 +2,15 @@
 
 This is a codec plugin for [Logstash](https://github.com/elastic/logstash) to parse protobuf messages.
 
-# Prerequisites
+# Prerequisites and Installation
  
 * prepare your ruby versions of the protobuf definitions, for example using the ruby-protoc compiler from https://github.com/codekitchen/ruby-protocol-buffers
-* download the logstash-codec-protobuf-$VERSION.gem to your computer.
+* download the [gem file](https://rubygems.org/gems/logstash-codec-protobuf) to your computer.
 * Install the plugin. From within your logstash directory, do
 	bin/plugin install /path/to/logstash-codec-protobuf-$VERSION.gem
 * use the codec in your logstash config file. See details below.
 
-# Usage
+## Usage
 
 Use this as a codec in any logstash input. Just provide the name of the class that your incoming objects will be encoded in, and specify the path to the compiled definition.
 Here's an example for a kafka input:
@@ -46,20 +46,20 @@ Set the class name to the parent class:
 	
 	class_name => "Foods::Dairy::Cheese"
 
-# Configuration
+## Configuration
 
 include_path	(required): an array of strings with filenames or directory names where logstash can find your protobuf definitions. Please provide absolute paths. For directories it will only try to import files ending on .rb
 
 class_name		(required): the name of the protobuf class that is to be decoded.
 
 
-# Troubleshooting
+## Troubleshooting
 
-## "uninitialized constant SOME_CLASS_NAME"
+### "uninitialized constant SOME_CLASS_NAME"
 
 If you include more than one definition class, consider the order of inclusion. This is especially relevant if you include whole directories. A definition might refer to another definition that is not loaded yet. In this case, please specify the files in the include_path variable in reverse order of reference. See 'Example with referenced definitions' above.
 
 
-# Roapmap
+# #Roadmap
 
 Currently the plugin supports the decode functionality only. Maybe we'll add the encoding part also.

data/lib/logstash/codecs/protobuf.rb

@@ -1,10 +1,10 @@
 # encoding: utf-8
-require "logstash/codecs/base"
-require "logstash/util/charset"
+require 'logstash/codecs/base'
+require 'logstash/util/charset'
 require 'protocol_buffers' # https://github.com/codekitchen/ruby-protocol-buffers
 
 class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
-  config_name "protobuf"
+  config_name 'protobuf'
 
   # Required: list of strings containing directories or files with protobuf definitions
   config :include_path, :validate => :array, :required => true
@@ -12,100 +12,72 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   # Name of the class to decode
   config :class_name, :validate => :string, :required => true
 
-  # remove 'set_fields' field
-  config :remove_set_fields, :validate => :boolean, :default => true # TODO add documentation
-
   
-  public
   def register
-    @include_path.each{|path| require_pb_path(path) }
-    @obj = create_object_from_name(@class_name)
-    
+    include_path.each { |path| require_pb_path(path) }
+    @obj = create_object_from_name(class_name)
+    @logger.debug("Protobuf files successfully loaded.")
   end
 
+  def decode(data)
+    decoded = @obj.parse(data.to_s)
+    results = extract_vars(decoded)
+    yield LogStash::Event.new(results) if block_given?
+  end # def decode
+
+  def encode(event)
+    raise 'Encodeør function not implemented yet for protobuf codec. Sorry!'
+    # @on_event.call(event, event.to_s)
+    # TODO integrate
+  end # def encode
+
+
 
   private
   def create_object_from_name(name)
     begin
-      c = name.split("::").inject(Object) { |n,c| n.const_get c }
-      return c
-    end
+      @logger.debug("Creating instance of " + name)
+      return name.split('::').inject(Object) { |n,c| n.const_get c }
+     end
   end
 
-  private 
-  def debug(message)
-    begin
-      if @debug 
-        @logger.debug(message)
-      end
-    end
-  end
 
-  private 
   def require_pb_path(dir_or_file)
-    f = dir_or_file.end_with? (".rb")
+    f = dir_or_file.end_with? ('.rb')
     begin
       if f
+        @logger.debug("Including protobuf file: " + dir_or_file)
         require dir_or_file
       else 
-        Dir[ dir_or_file + "/*.rb"].each { |file| 
+        Dir[ dir_or_file + '/*.rb'].each { |file|
+          @logger.debug("Including protobuf path: " + dir_or_file + "/" + file)
           require file 
         }
       end
     end
   end
 
-  private
-  def extract_variables(decoded_object)
-    begin
-      result_hash = {}
-      if decoded_object.nil?
-        return result_hash
-      end
-      decoded_object.instance_variables.map do |ivar|
-        varname = ivar.to_s
-        key = varname.gsub("@","")
-        value = decoded_object.instance_variable_get("#{ivar}")
-        recurse = value.is_a? (::ProtocolBuffers::Message)  # object is not a primitive scalar
-        is_iterable |= value.is_a? Enumerable # value.respond_to? :each # object is an array
-        if recurse
-          result_hash[key] = extract_variables(value)
-        elsif is_iterable
-          tmp = []
-          value.each do | nested |
-            tmp.push(nested)
-          end
-          result_hash[key] = tmp
+
+  def extract_vars(decoded_object)
+    return {} if decoded_object.nil?
+    results = {}
+    decoded_object.instance_variables.each do |key|
+      formatted_key = key.to_s.gsub('@', '')
+      next if (formatted_key == :set_fields || formatted_key == "set_fields")
+      instance_var = decoded_object.instance_variable_get(key)
+
+      results[formatted_key] =
+        if instance_var.is_a?(::ProtocolBuffers::Message) 
+          extract_vars(instance_var)
+        elsif instance_var.is_a?(::Hash)
+          instance_var.inject([]) { |h, (k, v)| h[k.to_s] = extract_vars(v); h }
+        elsif instance_var.is_a?(Enumerable) # is a list
+          instance_var.inject([]) { |h, v| h.push(extract_vars(v)); h }
         else
-          result_hash[key] = value
+          instance_var
         end
-      end
-      if remove_set_fields
-        return result_hash.except("set_fields") # todo try to do this before the object is handed to this method
-      else
-        return result_hash
-      end      
-    end
+     end
+   results
   end
 
-
-
-
-
-  public
-  def decode(data)
-    decoded = @obj.parse(data.to_s)
-    results = extract_variables(decoded)
-    event = LogStash::Event.new(results)  
-    yield event
-    
-  end # def decode
-
-  public
-  def encode(event)
-    raise "Encoder function not implemented yet for protobuf codec. Sorry!"
-    # @on_event.call(event, event.to_s)
-    # TODO integrate
-  end # def encode
-
 end # class LogStash::Codecs::Protobuf

data/logstash-codec-protobuf.gemspec

@@ -1,11 +1,12 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '0.1.0'
+  s.version         = '0.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors         = ["trivago"]
+  s.authors         = ["Inga Feick"]
+  s.email           = 'inga.feick@trivago.com'
   s.require_paths   = ["lib"]
 
   # Files

data/spec/codecs/protobuf_spec.rb

@@ -17,14 +17,12 @@ describe LogStash::Codecs::Protobuf do
     
       data = {:colour => 'rainbow', :horn_length => 18, :last_seen => 1420081471}
       unicorn = Animal::Unicorn.new(data)
-          
+        
       plugin.decode(unicorn.serialize_to_string) do |event|
-        insist { event.is_a? LogStash::Event }
-        insist { event["colour"] } == data[:colour]
-        insist { event["horn_length"] } == data[:horn_length]
-        insist { event["last_seen"] } == data[:last_seen]
+
+        expect(event["colour"] ).to eq(data[:colour] )
+        expect(event["horn_length"] ).to eq(data[:horn_length] )
       end
     end
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
-- trivago
+- Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-05 00:00:00.000000000 Z
+date: 2016-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -59,7 +59,7 @@ dependencies:
   prerelease: false
   type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
-email:
+email: inga.feick@trivago.com
 executables: []
 extensions: []
 extra_rdoc_files: []