logstash-codec-es_bulk 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 056be7360d412c0dc082aee477b91bbd3f9826fa
+  data.tar.gz: 1b86efe8e257a10e308bc43e5fa189fcb212fb92
+SHA512:
+  metadata.gz: 8023e5e0ea12fa1e83dc845bd1020c58e57007ec6a8c79b5835f22e6ae33539d405f2ffb13e363f5b9113ca0da552578788c510c1e1c1ec145fc498c5f259d19
+  data.tar.gz: b44553d7e61caec35d6227e3a903af593c540787aaa00759e326ee596b9076f43ab5e95f1f912d8b4031bd8703c327f9605548286342bb089a1db7d623b352f0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,6 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+require "logstash/devutils/rake"

data/lib/logstash/codecs/es_bulk.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/line"
+require "logstash/json"
+
+# This codec will decode the Elasticsearch bulk format into
+# individual events, plus metadata into the `@metadata` field.
+# 
+# Encoding is not supported at this time as the Elasticsearch
+# output submits Logstash events in bulk format.
+class LogStash::Codecs::ESBulk < LogStash::Codecs::Base
+  config_name "es_bulk"
+
+  milestone 1
+
+  public
+  def initialize(params={})
+    super(params)
+    @lines = LogStash::Codecs::Line.new
+    @lines.charset = "UTF-8"
+  end
+
+  public
+  def decode(data)
+    state = :initial
+    metadata = Hash.new
+    @lines.decode(data) do |bulk|
+      begin
+        line = LogStash::Json.load(bulk["message"])
+        case state
+        when :metadata
+          event = LogStash::Event.new(line)
+          event["@metadata"] = metadata
+          yield event
+          state = :initial
+        when :initial
+          metadata = line[line.keys[0]]
+          metadata["action"] = line.keys[0].to_s
+          state = :metadata
+          if line.keys[0] == 'delete'
+            event = LogStash::Event.new()
+            event["@metadata"] = metadata
+            yield event
+            state = :initial
+          end
+        end
+      rescue LogStash::Json::ParserError => e
+        @logger.error("JSON parse failure. ES Bulk messages must in be UTF-8 JSON", :error => e, :data => data)
+      end
+    end
+  end # def decode
+
+end # class LogStash::Codecs::ESBulk

data/logstash-codec-es_bulk.gemspec

@@ -0,0 +1,27 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-codec-es_bulk'
+  s.version         = '0.1.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This codec decodes messages received in the Elasticsearch bulk format"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'aaron.mildenstein@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'logstash-codec-line'
+
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/codecs/es_bulk_spec.rb

@@ -0,0 +1,57 @@
+require "logstash/codecs/es_bulk"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::ESBulk do
+  subject do
+    next LogStash::Codecs::ESBulk.new
+  end
+
+  context "#decode" do
+    it "should return 4 events from json data" do
+      data = <<-HERE
+      { "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
+      { "field1" : "value1" }
+      { "delete" : { "_index" : "test", "_type" : "type1", "_id" : "2" } }
+      { "create" : { "_index" : "test", "_type" : "type1", "_id" : "3" } }
+      { "field1" : "value3" }
+      { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1"} }
+      { "doc" : {"field2" : "value2"} }
+      HERE
+
+      count = 0
+      subject.decode(data) do |event|
+        case count
+        when 0
+          insist { event['@metadata']['_id'] } == "1"
+          insist { event['@metadata']['action'] } == "index"
+          insist { event['field1'] } == "value1"
+        when 1
+          insist { event['@metadata']['_id'] } == "2"
+          insist { event['@metadata']['action'] } == "delete"
+        when 2
+          insist { event['@metadata']['_id'] } == "3"
+          insist { event['@metadata']['action'] } == "create"
+          insist { event['field1'] } == "value3"
+        when 3
+          insist { event['@metadata']['_id'] } == "1"
+          insist { event['@metadata']['action'] } == "update"
+          insist { event['doc']['field2'] } == "value2"
+        end
+        count += 1
+      end
+      insist { count } == 4
+    end
+  end
+
+  context "fail to process non-bulk event then continue" do
+    it "continues after a fail" do
+      decoded = false
+      subject.decode("something that isn't a bulk event\n") do |event|
+        decoded = true
+      end
+      insist { decoded } == false
+    end
+  end
+
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-es_bulk
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-line
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: aaron.mildenstein@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/codecs/es_bulk.rb
+- logstash-codec-es_bulk.gemspec
+- spec/codecs/es_bulk_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: codec
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.4
+signing_key:
+specification_version: 4
+summary: This codec decodes messages received in the Elasticsearch bulk format
+test_files:
+- spec/codecs/es_bulk_spec.rb