logstash-codec-bulk_es 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0dec955882f2ef0b9ebbfa1da8aa301a98f661b9
-  data.tar.gz: d8f5a5efb014f6975dbeeb8fbe1ae25122712f21
+  metadata.gz: 6c64ed6ff5ea89415ac54393dbb4d2be66b93382
+  data.tar.gz: 90ed02febaed6611c1b7f10d7385b00d074469be
 SHA512:
-  metadata.gz: a6b2ec95d82713a1e637a404c47e2b237f6ca45a442c29ea6cf2cc785983abff18379ac9d089e2678adc45732f4a636cd55a3f227c4b6c2b62765d6514d330c1
-  data.tar.gz: 006e60a7a45f795220242fcebc85901a499d296d8f789232f76344bf10f0ec4a0fb841ade1bf704d833b40b64eeaef3bfd965b3998bdedc058513f74cfe78f35
+  metadata.gz: 92e69ca6e9d46f84a3a0d92bdf1be0f721c773825838edcfbd70e0355beca55f7909cd057804bddd311b0ded1154c6388429f3874289e02c28dcbecb911a9220
+  data.tar.gz: 7fdcc6ba178a4d327b1503cfe33b6f3581213e750e412d08a4df468350a2368a8d57921d3e21ee7fa23bc68195497661bedf5713afcad6b83870792a8ca546c8

data/lib/logstash/codecs/bulk_es.rb

@@ -34,14 +34,22 @@ class LogStash::Codecs::BulkEs < LogStash::Codecs::Base
         case @state
         when :metadata
           if @metadata["action"] == 'update'
-             if line.has_key?("doc")
-               event = LogStash::Event.new(line["doc"])
-             elsif
-               event = LogStash::Event.new(line)
-             end
-          elsif
+            if line.has_key?("doc")
+              event = LogStash::Event.new(line["doc"])
+              if line.has_key?("doc_as_upsert")
+                @metadata["doc_as_upsert"] = line["doc_as_upsert"]
+              end
+            else
+              event = LogStash::Event.new(line)
+            end
+
+            if line.has_key?("upsert")
+              @metadata["upsert"] = LogStash::Json.dump(line["upsert"])
+            end
+          elsif #for action = index or create
             event = LogStash::Event.new(line)
           end
+
           event.set("@metadata", @metadata)
           yield event
           @state = :init

data/logstash-codec-bulk_es.gemspec

@@ -1,17 +1,17 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-codec-bulk_es'
-  s.version       = '0.1.2'
+  s.version       = '0.1.3'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'This codec decodes messages received in the Elasticsearch bulk format.'
   s.description   = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'
   s.homepage      = 'http://www.elastic.co/guide/en/logstash/current/index.html'
-  s.authors       = ['']
+  s.authors       = ['saran.ramjeet@gmail.com']
   s.email         = 'saran.ramjeet@gmail.com'
   s.require_paths = ['lib']
 
   # Files
   s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
-   # Tests
+  # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin

data/spec/codecs/bulk_es_spec.rb

@@ -1,3 +1,73 @@
 # encoding: utf-8
-require_relative '../spec_helper'
-require "logstash/codecs/bulk_es"
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/es_bulk"
+require "logstash/event"
+require "insist"
+
+describe LogStash::Codecs::ESBulk do
+  subject do
+    next LogStash::Codecs::ESBulk.new
+  end
+
+  context "#decode" do
+    it "should return 6 events from json data" do
+      data = <<-HERE
+      { "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
+      { "field1" : "value1" }
+      { "delete" : { "_index" : "test", "_type" : "type1", "_id" : "2" } }
+      { "create" : { "_index" : "test", "_type" : "type1", "_id" : "3" } }
+      { "field1" : "value3" }
+      { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1"} }
+      { "doc" : {"field2" : "value2"} }
+      { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1"} }
+      { "doc" : {"field2" : "value4"}, "upsert": { "field2": "value4"} }
+      { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1"} }
+      { "doc" : {"field2" : "value5"}, "doc_as_upsert": true }
+      HERE
+
+      count = 1
+      subject.decode(data) do |event|
+        case count
+        when 1
+          insist { event.get("[@metadata][_id]") } == "1"
+          insist { event.get("[@metadata][action]") } == "index"
+          insist { event.get("field1") } == "value1"
+        when 2
+          insist { event.get("[@metadata][_id]") } == "2"
+          insist { event.get("[@metadata][action]") } == "delete"
+        when 3
+          insist { event.get("[@metadata][_id]") } == "3"
+          insist { event.get("[@metadata][action]") } == "create"
+          insist { event.get("field1") } == "value3"
+        when 4
+          insist { event.get("[@metadata][_id]") } == "1"
+          insist { event.get("[@metadata][action]") } == "update"
+          insist { event.get("field2") } == "value2"
+        when 5
+          insist { event.get("[@metadata][_id]") } == "1"
+          insist { event.get("[@metadata][action]") } == "update"
+          insist { event.get("field2") } == "value4"
+          insist { event.get("[@metadata][upsert]") } == "{\"field2\":\"value4\"}"
+        when 6
+          insist { event.get("[@metadata][_id]") } == "1"
+          insist { event.get("[@metadata][action]") } == "update"
+          insist { event.get("field2") } == "value5"
+          insist { event['@metadata']['doc_as_upsert'] } == true
+        end
+        count += 1
+      end
+      insist { count } == 6
+    end
+  end
+
+  context "fail to process non-bulk event then continue" do
+    it "continues after a fail" do
+      decoded = false
+      subject.decode("something that isn't a bulk event\n") do |event|
+        decoded = true
+      end
+      insist { decoded } == false
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-bulk_es
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
-- ''
+- saran.ramjeet@gmail.com
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-06 00:00:00.000000000 Z
+date: 2017-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -62,7 +62,6 @@ extra_rdoc_files: []
 files:
 - lib/logstash/codecs/bulk_es.rb
 - spec/codecs/bulk_es_spec.rb
-- spec/spec_helper.rb
 - logstash-codec-bulk_es.gemspec
 - CHANGELOG.md
 - DEVELOPER.md
@@ -98,4 +97,3 @@ specification_version: 4
 summary: This codec decodes messages received in the Elasticsearch bulk format.
 test_files:
 - spec/codecs/bulk_es_spec.rb
-- spec/spec_helper.rb

data/spec/spec_helper.rb

@@ -1,2 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"