logstash-codec-protobuf 1.0.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43ef1232f802b173bf7fee33f894c31145e5032e
-  data.tar.gz: f0d955aeee52c1bbb6f72abe016603c6eac8994e
+  metadata.gz: 826cf27e43ea66cbabc39b226b229d179fd275d6
+  data.tar.gz: 0e2888f0622344cc8dddf20b03b9c89974d0613f
 SHA512:
-  metadata.gz: fb9a3a3b646935acd74d986e3defd67a79ccb14bad91bbd9abe4240bf938ef001a709a8ec05efa243410c25d08e3577a1c3a53e1614bccda55d0a6f07837e719
-  data.tar.gz: ad7d0690b3169ab22623db7b5e8dfd5a17d2cb1e082d54c39da86fef8239dd3576a1d1ed5894dcd27f3fe332f1538d8a71a960cb41a7ee85ab7f5f05c32dc4ef
+  metadata.gz: 8f49b8820a8a70a686a42c7561621946a426e74f839f18476c84a1e59dc06e2b71cbb2045c8281fb36d6cd432ee22ea896900dbbacf01686b46530b8008a8726
+  data.tar.gz: a162df9db2fed47938ef1cdd2901c28230a6f127351f464288582561b2d0fd5baef1fa59da91af503e2cd1b36e598932af827f0492e8becd5b9e122c19494828

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 1.0.1
+ - Speed improvement, better exception handling and code refactoring
+
 ## 1.0.0
  - Update to v5.0 API
 
-## 0.1.2
- - First version of this plugin
+## 0.1.2
+ - First version of this plugin

data/Gemfile

@@ -1,3 +1,11 @@
 source 'https://rubygems.org'
-gem 'logstash-docgen', :path => "/Users/suyog/ws/elastic/logstash/tools/logstash-docgen"
-gemspec
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,106 @@
+:plugin: protobuf
+:type: codec
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Protobuf codec plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This codec converts protobuf encoded messages into logstash events and vice versa. 
+
+Requires the protobuf definitions as ruby files. You can create those using the [ruby-protoc compiler](https://github.com/codekitchen/ruby-protocol-buffers).
+
+The following shows a usage example for decoding events from a kafka stream:
+[source,ruby]
+kafka 
+{
+ zk_connect => "127.0.0.1"
+ topic_id => "your_topic_goes_here"
+ codec => protobuf 
+ {
+   class_name => "Animal::Unicorn"
+   include_path => ['/path/to/protobuf/definitions/UnicornProtobuf.pb.rb']
+ }
+}
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Protobuf Codec Configuration Options
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-class_name>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-include_path>> |<<array,array>>|Yes
+|=======================================================================
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-class_name"]
+===== `class_name` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Name of the class to decode.
+If your protobuf definition contains modules, prepend them to the class name with double colons like so:
+[source,ruby]
+class_name => "Foods::Dairy::Cheese"
+
+This corresponds to a protobuf definition starting as follows:
+[source,ruby]
+module Foods
+   module Dairy
+       class Cheese
+           # here are your field definitions.
+
+If your class references other definitions: you only have to add the main class here.
+
+[id="plugins-{type}s-{plugin}-include_path"]
+===== `include_path` 
+
+  * This is a required setting.
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+List of absolute pathes to files with protobuf definitions. 
+When using more than one file, make sure to arrange the files in reverse order of dependency so that each class is loaded before it is 
+refered to by another.
+
+Example: a class _Cheese_ referencing another protobuf class _Milk_
+[source,ruby]
+module Foods
+  module Dairy
+        class Cheese
+           set_fully_qualified_name "Foods.Dairy.Cheese"
+           optional ::Foods::Cheese::Milk, :milk, 1
+           optional :int64, :unique_id, 2
+           # here be more field definitions
+
+would be configured as
+[source,ruby]
+include_path => ['/path/to/protobuf/definitions/Milk.pb.rb','/path/to/protobuf/definitions/Cheese.pb.rb']
+
+When using the codec in an output plugin: 
+* make sure to include all the desired fields in the protobuf definition, including timestamp. 
+  Remove fields that are not part of the protobuf definition from the event by using the mutate filter.
+* the @ symbol is currently not supported in field names when loading the protobuf definitions for encoding. Make sure to call the timestamp field "timestamp" 
+  instead of "@timestamp" in the protobuf file. Logstash event fields will be stripped of the leading @ before conversion.
+ 
+
+

data/lib/logstash/codecs/protobuf.rb

@@ -66,32 +66,23 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
   config :include_path, :validate => :array, :required => true
 
 
-
-
-
   def register
     @pb_metainfo = {}
     include_path.each { |path| require_pb_path(path) }
     @obj = create_object_from_name(class_name)
     @logger.debug("Protobuf files successfully loaded.")
-
   end
 
-  def decode(data)
-    decoded = @obj.parse(data.to_s)
-    results = keys2strings(decoded.to_hash)
-    yield LogStash::Event.new(results) if block_given?
-  end # def decode
 
-  def keys2strings(data)
-    if data.is_a?(::Hash)
-      new_hash = Hash.new
-      data.each{|k,v| new_hash[k.to_s] = keys2strings(v)}
-      new_hash
-    else
-      data
+  def decode(data)
+    begin
+      decoded = @obj.parse(data.to_s)
+      yield LogStash::Event.new(decoded.to_hash) if block_given?
+    rescue => e
+      @logger.warn("Couldn't decode protobuf: #{e.inspect}.")
+      # raise e
     end
-  end
+  end # def decode
 
 
   def encode(event)
@@ -99,68 +90,53 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     @on_event.call(event, protobytes)
   end # def encode
 
+
   private
-  def generate_protobuf(event)  
-    meth = self.method("encoder_strategy_1")
-    data = meth.call(event, @class_name)
+  def generate_protobuf(event)
     begin
+      data = _encode(event, @class_name)
       msg = @obj.new(data)
       msg.serialize_to_string
     rescue NoMethodError
       @logger.debug("error 2: NoMethodError. Maybe mismatching protobuf definition. Required fields are: " + event.to_hash.keys.join(", "))
+    rescue => e
+      @logger.debug("Couldn't generate protobuf: ${e}")
     end
   end
 
-  def encoder_strategy_1(event, class_name)
-    _encoder_strategy_1(event.to_hash, class_name)
 
-  end
-
-  def _encoder_strategy_1(datahash, class_name)
-    fields = clean_hash_keys(datahash)
-    fields = flatten_hash_values(fields) # TODO we could merge this and the above method back into one to save one iteration, but how are we going to name it?
+  def _encode(datahash, class_name)
+    fields = prepare_for_encoding(datahash)    
     meta = get_complex_types(class_name) # returns a hash with member names and their protobuf class names
     meta.map do | (k,typeinfo) |
       if fields.include?(k)
         original_value = fields[k] 
         proto_obj = create_object_from_name(typeinfo)
         fields[k] = 
-          if original_value.is_a?(::Array) 
-            ecs1_list_helper(original_value, proto_obj, typeinfo)
-            
+          if original_value.is_a?(::Array)
+            # make this field an array/list of protobuf objects
+            # value is a list of hashed complex objects, each of which needs to be protobuffed and
+            # put back into the list.
+            original_value.map { |x| _encode(x, typeinfo) } 
+            original_value
           else 
-            recursive_fix = _encoder_strategy_1(original_value, class_name)
+            recursive_fix = _encode(original_value, class_name)
             proto_obj.new(recursive_fix)
           end # if is array
       end
-
-    end 
-    
+    end    
     fields
   end
 
-  def ecs1_list_helper(value, proto_obj, class_name)
-    # make this field an array/list of protobuf objects
-    # value is a list of hashed complex objects, each of which needs to be protobuffed and
-    # put back into the list.
-    next unless value.is_a?(::Array)
-    value.map { |x| _encoder_strategy_1(x, class_name) } 
-    value
-  end
 
-  def flatten_hash_values(datahash)
+  def prepare_for_encoding(datahash)
+    # the data cannot be encoded until certain criteria are met:
+    # 1) remove @ signs from keys 
     # 2) convert timestamps and other objects to strings
-    next unless datahash.is_a?(::Hash)
-    
-    ::Hash[datahash.map{|(k,v)| [k, (convert_to_string?(v) ? v.to_s : v)] }]
+    next unless datahash.is_a?(::Hash)    
+    ::Hash[datahash.map{|(k,v)| [remove_atchar(k.to_s), (convert_to_string?(v) ? v.to_s : v)] }]
   end
 
-  def clean_hash_keys(datahash)
-    # 1) remove @ signs from keys 
-    next unless datahash.is_a?(::Hash)
-    
-    ::Hash[datahash.map{|(k,v)| [remove_atchar(k.to_s), v] }]
-  end #clean_hash_keys
 
   def convert_to_string?(v)
     !(v.is_a?(Fixnum) || v.is_a?(::Hash) || v.is_a?(::Array) || [true, false].include?(v))
@@ -171,7 +147,7 @@ class LogStash::Codecs::Protobuf < LogStash::Codecs::Base
     key.dup.gsub(/@/,'')
   end
 
-  private
+  
   def create_object_from_name(name)
     begin
       @logger.debug("Creating instance of " + name)

data/logstash-codec-protobuf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-protobuf'
-  s.version         = '1.0.0'
+  s.version         = '1.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This codec may be used to decode (via inputs) and encode (via outputs) protobuf messages"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths   = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-protobuf
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.2
 platform: ruby
 authors:
 - Inga Feick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-04 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,6 +71,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/codecs/protobuf.rb
 - logstash-codec-protobuf.gemspec
 - spec/codecs/protobuf_spec.rb