logstash-codec-transit 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ce040e485956a2c8925d9b88eb5756b7de0a4857
+  data.tar.gz: 2dd9f9b78f0b041a18bd6346c1d5d31353600b45
+SHA512:
+  metadata.gz: 30230d37f665cf9e64e148980a3143683c921545aa3ba0a5d307b5df1f13bfd5a71e795cf0a4fe972ddf48e689db1611384a5715b7ade41be1890f4ec7104aa2
+  data.tar.gz: f0dff14b3816e425cac503a00a62df7d855ebc6b37964ac30a01efadb451f3455ece87cf1a6d235aaa594adea3fe1e7c3e23c695f1738869612d4e4f7e04bccd

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+Jarfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+jruby-1.7.21

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Agents' Mutual <http://www.agentsmutual.co.uk>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,25 @@
+# Logstash Transit Plugin
+
+A codec for logstash that encodes and decodes messages being sent using the [transit](http://github.com/cognitect/transit-format) format.
+
+## Install
+
+`$LOGSTASH_HOME/bin/plugin install transit`
+
+## Configure
+
+The transport parameter can be either `json` (default) or `msgpack`. There is
+also an option to base64 encode/decode the messages (defaults to `false`).
+
+
+### Example configuration
+```
+input {
+  stdin {
+    codec => transit {
+      transport => json
+      base64 => false
+    }
+  }
+}
+```

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/codecs/transit.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/timestamp"
+require "logstash/util"
+
+class LogStash::Codecs::Transit < LogStash::Codecs::Base
+  config_name "transit"
+
+  # Underlying transport to be used. Valid values are 'json' or 'msgpack'
+  config :transport, :validate => :string, :default => 'json'
+  # Whether messages are base64 encoded or not
+  config :base64, :validate => :boolean, :default => false
+
+  public
+  def register
+    require "transit"
+    require "base64" if @base64
+  end
+
+  public
+  def decode(data)
+    begin
+      data = Base64.decode64(data) if @base64
+      io = StringIO.new data
+      reader = Transit::Reader.new(@transport.to_sym, io)
+      event = LogStash::Event.new(reader.read)
+      event["tags"] ||= []
+    rescue => e
+      @logger.warn("Trouble parsing transit input, falling back to plain text",
+                   :input => data, :exception => e)
+      event = LogStash::Event.new("") unless event
+      event["message"] = data
+      event["tags"] ||= []
+      event["tags"] << "_transitparsefailure"
+    end
+    yield event
+  end
+
+  public
+  def encode(event)
+    data = LogStash::Util.normalize(event.to_hash)
+    io = StringIO.new
+    writer = Transit::Writer.new(@transport.to_sym, io)
+    writer.write(data.merge(LogStash::Event::TIMESTAMP => event.timestamp.to_iso8601))
+
+    encoded = io.string
+    encoded = Base64.encode64(encoded) if @base64
+    @on_event.call(event, encoded)
+  end
+
+end

data/logstash-codec-transit.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-codec-transit'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Encode and decode transit formatted data"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install transit. This gem is not a stand-alone program"
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths   = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "transit-ruby"
+
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/codecs/transit_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/transit"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::Transit do
+  def codec_test
+    data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "date" => DateTime.new(2015,2,3)}
+    event = LogStash::Event.new(data)
+    encoded = nil
+    subject.on_event do |e, d|
+      encoded = d
+      got_event = true
+    end
+    subject.encode(event)
+    insist { got_event }
+
+    subject.decode(encoded) do |event|
+      insist { event.is_a? LogStash::Event }
+      insist { event["foo"] } == data["foo"]
+      insist { event["baz"] } == data["baz"]
+      insist { event["bah"] } == data["bah"]
+      insist { event["date"] } == data["date"]
+    end
+  end
+
+  context "msgpack" do
+    subject do
+      LogStash::Codecs::Transit.new("transport" => "msgpack")
+    end
+
+    context "encode + decode" do
+      it "should return an event from transit data" do
+        codec_test
+      end
+    end
+  end
+
+  context "json" do
+    subject do
+      LogStash::Codecs::Transit.new("transport" => "json")
+    end
+
+    context "encode + decode" do
+      it "should return an event from transit data" do
+        codec_test
+      end
+    end
+  end
+
+  context "base64 msgpack" do
+    subject do
+      LogStash::Codecs::Transit.new("transport" => "msgpack", "base64" => true)
+    end
+
+    context "encode + decode" do
+      it "should return an event from transit data" do
+        codec_test
+      end
+    end
+  end
+end

data/spec/test-codec.conf

@@ -0,0 +1,15 @@
+input {
+  stdin {
+    codec => transit {
+      transport => msgpack
+      base64 => false
+    }
+  }
+}
+output {
+  stdout {
+    codec => transit {
+      transport => json
+    }
+  }
+}

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-transit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-08-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: transit-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install transit. This gem is not a stand-alone program
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .ruby-version
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/codecs/transit.rb
+- logstash-codec-transit.gemspec
+- spec/codecs/transit_spec.rb
+- spec/test-codec.conf
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Encode and decode transit formatted data
+test_files:
+- spec/codecs/transit_spec.rb
+- spec/test-codec.conf