logstash-codec-json_stream 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b844cfe3444a066af5ab9b7c0da943da2822cdd490c4b5c9bee8c2889c95cb3c
-  data.tar.gz: a40e7766d69b01e601fdf4231b013c65c1db10e8e824b1ab2ff9fafa25dab5b6
+  metadata.gz: 61d4c637a37d172a265d32cf5adeeeb84b7e7d6a821c3c4a52d5613b9e7fc28e
+  data.tar.gz: d821b50785f069a7402a1f757bf7ee3e35229ae75184def3b60707a6defb581f
 SHA512:
-  metadata.gz: cd984ba16e049cfedd692ad99f76462acc2ff493b359a8e3a09b8530af0cb1fcec49ef00d2823a7b5fe2221a37536b237275b89514a2fcc3da64173c47ba7636
-  data.tar.gz: 75252622493230be43d863a66e99a7e3fbb1f219d0f1bf8571c4f6686fe780825b1b5eaa497de6b45516a04bf143b928c3f1168fed623f2dce272d7bbf9e3fc3
+  metadata.gz: aaef763e7882036ff02b4f9215b3aed538f90b07a0498464b1e9f546646744e981a23de6d9c87b45b76462235cf7f799d07f14cc1955a3ce4543013fbee89fe3
+  data.tar.gz: f6765f37c81b588e83830e1f99dae4e6941ba591051bdc1bbeb59b7d26023cde86173d046e44e4424c2b2f4089e79151c71400b6f5f689676926ff24c86237bb

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.1.0
+- Refactor code
+- Json Parse Failure should be a warning level
 ## 1.0.0
 - Add spec tests thanks to johannesthoenes4000 & thomasklinger123
 - Add a better parsing method thanks to johannesthoenes4000

data/lib/logstash/codecs/json_stream.rb

@@ -21,9 +21,10 @@ class LogStash::Codecs::JSONStream < LogStash::Codecs::Base
   end
 
   def decode(concatenated_json, &block)
-    decode_unsafe(concatenated_json, &block)
+    array_json = @converter.convert("[#{concatenated_json.gsub('}{', '},{')}]")
+    parse(array_json, &block)
   rescue LogStash::Json::ParserError => e
-    @logger.error("JSON parse error for json stream / concatenated json, original data now in message field", :error => e, :data => concatenated_json)
+    @logger.warn("JSON parse error for json stream / concatenated json, original data now in message field", :error => e, :data => concatenated_json)
     yield LogStash::Event.new("message" => concatenated_json, "tags" => ["_jsonparsefailure"])
   rescue StandardError => e
     # This should NEVER happen. But hubris has been the cause of many pipeline breaking things
@@ -38,18 +39,30 @@ class LogStash::Codecs::JSONStream < LogStash::Codecs::Base
   end
 
   def encode(event)
-    @logger.error("Encoding is not supported by 'concatenated_json' plugin yet")
+    @logger.error("Encoding is not supported by 'jsonstream' plugin yet")
   end
 
-  def flush(&block)
-    @logger.debug("empty flush method -- nothing to do")
+  private
+
+  # from_json_parse uses the Event#from_json method to deserialize and directly produce events
+  def from_json_parse(json, &block)
+    LogStash::Event.from_json(json).each { |event| yield event }
+  rescue LogStash::Json::ParserError => e
+    @logger.warn("JSON parse error, original data now in message field", :error => e, :data => json)
+    yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"])
   end
 
-  private
-  def decode_unsafe(concatenated_json)
-    array_json = @converter.convert("[#{concatenated_json.gsub('}{', '},{')}]")
-    LogStash::Json.load(array_json).each do |decoded_event|
-       yield(LogStash::Event.new(decoded_event))
-    end
+  # legacy_parse uses the LogStash::Json class to deserialize json
+  def legacy_parse(json, &block)
+    # ignore empty/blank lines which LogStash::Json#load returns as nil
+    o = LogStash::Json.load(json)
+    yield(LogStash::Event.new(o)) if o
+  rescue LogStash::Json::ParserError => e
+    @logger.warn("JSON parse error, original data now in message field", :error => e, :data => json)
+    yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"])
   end
+
+  # keep compatibility with all v2.x distributions. only in 2.3 will the Event#from_json method be introduced
+  # and we need to keep compatibility for all v2 releases.
+  alias_method :parse, LogStash::Event.respond_to?(:from_json) ? :from_json_parse : :legacy_parse
 end

data/logstash-codec-json_stream.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-json_stream'
-  s.version         = '1.0.0'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads and writes non JSON Streams"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,5 +25,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-line', '>= 2.1.0'
 
   s.add_development_dependency 'logstash-devutils'
+
 end
 

data/spec/codecs/json_stream_spec.rb

@@ -8,11 +8,23 @@ require "insist"
 describe LogStash::Codecs::JSONStream do
 
   class LogStash::Codecs::JSONStream
-    public :decode_unsafe # use method without error logging for better visibility of errors
+    public :from_json_parse # use method without error logging for better visibility of errors
   end
 
   let(:codec_options) { {} }
 
+  context "#decode" do
+    it "should return an event from json data" do
+      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
+      subject.decode(LogStash::Json.dump(data)) do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event.get("foo") } == data["foo"]
+        insist { event.get("baz") } == data["baz"]
+        insist { event.get("bah") } == data["bah"]
+      end
+    end
+  end
+
   context "default parser choice" do
     subject do
       LogStash::Codecs::JSONStream.new(codec_options)
@@ -51,12 +63,11 @@ EOS
     end
 
     it "should read multiple events from data" do
-      events = events_from_file('log-stream.valid-line-formatted')
-      insist { events.size } == 5
+      events = events_from_file('log-stream.real-formatted')
+      insist { events.size } == 4
 
       events.each do |event|
         insist { event.is_a? LogStash::Event }
-        insist { event.get("logGroup") } == "test-core"
         insist { event.get("messageType") } == "DATA_MESSAGE"
         insist { event.get("logEvents").size } != 0
         event.get("logEvents").each do |event|
@@ -85,7 +96,7 @@ EOS
   def events_from_string data
     events = []
     data_without_formatting = data.gsub(/(\n|\s{2,})/, '')
-    subject.decode_unsafe(data_without_formatting) { |event| events << event }
+    subject.decode(data_without_formatting) { |event| events << event }
     events
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-json_stream
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-26 00:00:00.000000000 Z
+date: 2019-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement