logstash-codec-thrift 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ee6ee6cc8ec8d720b7910442c5ed7dc8a440742c
+  data.tar.gz: b02e5490937476c9eb152c2fcd78b0e6c7b8c89d
+SHA512:
+  metadata.gz: 6a418f2bf9254f31b4160305dfec168c3689135f1b300183524623f99743ea7c343669306f8d0fa4f24307f4686314bfb7a8f28fb53bf1e886b786f610bac434
+  data.tar.gz: 22e46011821616d46eea3de80bb7165dcd454bf58ddcc2ec47f09de3b0a3974ca056b31ccad8d27826d439193fc46cb051f92f2b5e261e1db37c4dc4385ed32b

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,43 @@
+# Logstash Codec Thrift
+
+## Install
+
+    $ ./bin/plugin install logstash-codec-thrift
+
+## Usage
+
+```ruby
+input {
+  zeromq {
+    codec => thrift {
+      classname => "example_thrift_class"
+      file => "/path/to/your/thrift/gen/ruby/example_thrift_class_types.rb"
+      protocol_factory => "JsonProtocolFactory" # optional, defalut: BinaryProtocolFactory
+    }
+  }
+}
+```
+
+## Install dependencies
+
+Install jruby:
+
+    $ brew install jruby
+
+Install jruby bundler:
+
+    $ jruby -S gem install bundler
+
+Install plugin dependencies:
+
+    $ jruby -S bundler install
+
+## Build & Install
+
+Build Gem:
+
+    $ gem build logstash-codec-thrift.gemspec
+
+Install Gem:
+
+    $ ./bin/plugin install /your/local/plugin/logstash-codec-thrift.gem

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/codecs/thrift-event.rb

@@ -0,0 +1,32 @@
+#
+# Autogenerated by Thrift Compiler (0.9.2)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+
+class ThriftEvent
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  TIMESTAMP = 1
+  VERSION = 2
+  HOST = 3
+  MESSAGE = 4
+
+  FIELDS = {
+    TIMESTAMP => {:type => ::Thrift::Types::STRING, :name => 'timestamp'},
+    VERSION => {:type => ::Thrift::Types::STRING, :name => 'version'},
+    HOST => {:type => ::Thrift::Types::STRING, :name => 'host', :optional => true},
+    MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message', :optional => true}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field timestamp is unset!') unless @timestamp
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field version is unset!') unless @version
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+

data/lib/logstash/codecs/thrift.rb

@@ -0,0 +1,109 @@
+# encoding: utf-8
+require "logger"
+require "logstash/codecs/base"
+require "logstash/namespace"
+require "logstash/timestamp"
+require "thrift"
+require "logstash/codecs/thrift-event"
+
+# Read serialized Thrift records as Logstash events
+#
+# This plugin is used to serialize Logstash events as 
+# Thrift objects, as well as deserializing Thrift objects into 
+# Logstash events.
+#
+# ==== Encoding
+# 
+# This codec is for serializing individual Logstash events as Thrift.
+#
+#
+# ==== Decoding
+#
+# This codec is for deserializing individual Thrift serialized objects as Logstash events.
+#
+#
+# ==== Usage
+# Example usage with 0mq input.
+#
+# [source,ruby]
+# ----------------------------------
+# input {
+#   zeromq {
+#     codec => thrift {
+#       classname => "example_class"
+#       file => "~/example_class.rb"
+#       protocol_factory => "JsonProtocolFactory"
+#     }
+#   }
+# }
+# filter {
+#   ...
+# }
+# output {
+#   ...
+# }
+# ----------------------------------
+class LogStash::Codecs::Thrift < LogStash::Codecs::Base
+
+    config_name "thrift"
+
+    # class name to serialize/deserialize against
+    config :classname, :validate => :string, :required => true
+    
+    # file defining @classname
+    config :file, :validate => :string, :required => true
+
+    # protocol factory to use for serialization/deserialization
+    #
+    # see http://www.rubydoc.info/gems/thrift/latest/Thrift/Serializer for more info
+    config :protocol_factory, :validate => :string, :required => false, :default => "BinaryProtocolFactory"
+    
+    public
+    def register
+        @logger.info("Initializing logstash thrift codec for class: " + @classname)
+        require @file
+        @clazz = Object.const_get(@classname.capitalize)
+        @protocolFactory = Thrift.const_get(@protocol_factory)
+        @logger.info("Done initializing logstash thrift codec!")
+    end
+
+    public
+    def decode(data)
+        deserializer = Thrift::Deserializer.new(@protocolFactory.new)
+        yield LogStash::Event.new(as_hash(deserializer.deserialize(@clazz.new, data)))
+    end
+
+    public
+    def encode(event)
+        serializer = Thrift::Serializer.new(@protocolFactory.new)
+        thriftEvent = ThriftEvent.new
+        thriftEvent.timestamp = event["@timestamp"].to_s
+        thriftEvent.version = event["@version"]
+        thriftEvent.host = event["host"]
+        thriftEvent.message = event["message"]
+        @on_event.call(event, serializer.serialize(thriftEvent))
+    end
+
+    private
+    def as_hash(obj)
+        hash = {}
+        obj.instance_variables.each { |var|
+            key = var.to_s.delete("@")
+            value = obj.instance_variable_get(var)
+            if value.is_a?(Array)
+                hash[key] = []
+                value.each do |elem|
+                    if elem.is_a?(Array) || elem.is_a?(Object)
+                        hash[key].push(as_hash(elem))
+                    else
+                        hash[key].push(elem)
+                    end
+                end
+            else
+                hash[key] = value
+            end
+        }
+        hash
+    end
+
+end

data/lib/thrift/event.thrift

@@ -0,0 +1,6 @@
+struct event {
+    1: required string timestamp, #timestamp as iso8601
+    2: required string version,
+    3: optional string host,
+    4: optional string message
+}

data/logstash-codec-thrift.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-codec-thrift'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This example input streams a string at a definable interval."
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Active Agent AG"]
+  s.email           = 'dev@active-agent.com'
+  s.homepage        = "http://active-agent.com"
+  s.require_paths   = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'thrift'
+
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.15'
+
+
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-thrift
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Active Agent AG
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: thrift
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.15
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.15
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: dev@active-agent.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- lib/logstash/codecs/thrift-event.rb
+- lib/logstash/codecs/thrift.rb
+- lib/thrift/event.thrift
+- logstash-codec-thrift.gemspec
+homepage: http://active-agent.com
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: This example input streams a string at a definable interval.
+test_files: []