logstash-codec-json_stream 0.0.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c0e43af36dd5cac97f6d09ef5cb84974e1d3892d
-  data.tar.gz: 518a4e627e794a2235f484efac865884b9ad1c92
+SHA256:
+  metadata.gz: b844cfe3444a066af5ab9b7c0da943da2822cdd490c4b5c9bee8c2889c95cb3c
+  data.tar.gz: a40e7766d69b01e601fdf4231b013c65c1db10e8e824b1ab2ff9fafa25dab5b6
 SHA512:
-  metadata.gz: 0c9686b8e0ef66c0a42456382ad4c0526b3807aeb3a39e52d0529e66f06519141b485b3d746027370c333ae8416c73d886d6e0c1c8dc133dd9462f47e1541e6f
-  data.tar.gz: 6786835c54bb2af7e91cfe41232595bcc5baa044633b6ae0b411698d366d9dc8243207e3a98d8aee032232878581c79135a38c6d6fdcd964140b293993e73cdf
+  metadata.gz: cd984ba16e049cfedd692ad99f76462acc2ff493b359a8e3a09b8530af0cb1fcec49ef00d2823a7b5fe2221a37536b237275b89514a2fcc3da64173c47ba7636
+  data.tar.gz: 75252622493230be43d863a66e99a7e3fbb1f219d0f1bf8571c4f6686fe780825b1b5eaa497de6b45516a04bf143b928c3f1168fed623f2dce272d7bbf9e3fc3

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.0.0
+- Add spec tests thanks to johannesthoenes4000 & thomasklinger123
+- Add a better parsing method thanks to johannesthoenes4000
 ## 0.0.1
 - Forked from Json_lines codec by elastic @elastic: Thank you
 - Added bracket counter to extract jsons.

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-codec-json_lines.svg)](https://travis-ci.org/logstash-plugins/logstash-codec-json_lines)
+[![Travis Build Status](https://travis-ci.org/cherweg/logstash-codec-json_stream.svg?branch=master)](https://travis-ci.org/cherweg/logstash-codec-json_stream)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/lib/logstash/codecs/json_stream.rb

@@ -1,20 +1,18 @@
 # encoding: utf-8
 require "logstash/codecs/base"
 require "logstash/util/charset"
-require "logstash/util/buftok"
 require "logstash/json"
 
+
 # This codec will decode streamed JSON that is not delimited.
 # Encoding will emit a single JSON string ending in a `@delimiter`
 
 class LogStash::Codecs::JSONStream < LogStash::Codecs::Base
+
   config_name "json_stream"
 
   config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
 
-  # Change the delimiter that separates lines
-  config :delimiter, :validate => :string, :default => "\n"
-
   public
 
   def register
@@ -22,33 +20,25 @@ class LogStash::Codecs::JSONStream < LogStash::Codecs::Base
     @converter.logger = @logger
   end
 
-  def decode(data, &block)
-    io = StringIO.new data
-
-    loop.inject(counter: 0, string: '') do |acc|
-      char = io.getc
-
-      break if char.nil? # EOF
-      next acc if acc[:counter].zero? && char != '{' # between objects
-
-      acc[:string] << char
-
-      if char == '}' && (acc[:counter] -= 1).zero?
-        # ⇓⇓⇓ # CALLBACK, feel free to JSON.parse here
-        parse(@converter.convert(acc[:string].gsub(/\p{Space}+/, ' ')), &block)
-        next {counter: 0, string: ''} # from scratch
-      end
-
-      acc.tap do |result|
-        result[:counter] += 1 if char == '{'
-      end
-    end
+  def decode(concatenated_json, &block)
+    decode_unsafe(concatenated_json, &block)
+  rescue LogStash::Json::ParserError => e
+    @logger.error("JSON parse error for json stream / concatenated json, original data now in message field", :error => e, :data => concatenated_json)
+    yield LogStash::Event.new("message" => concatenated_json, "tags" => ["_jsonparsefailure"])
+  rescue StandardError => e
+    # This should NEVER happen. But hubris has been the cause of many pipeline breaking things
+    # If something bad should happen we just don't want to crash logstash here.
+    @logger.error(
+      "An unexpected error occurred parsing JSON data",
+      :data => concatenated_json,
+      :message => e.message,
+      :class => e.class.name,
+      :backtrace => e.backtrace
+    )
   end
 
   def encode(event)
-    # Tack on a @delimiter for now because previously most of logstash's JSON
-    # outputs emitted one per line, and whitespace is OK in json.
-    @on_event.call(event, "#{event.to_json}#{@delimiter}")
+    @logger.error("Encoding is not supported by 'concatenated_json' plugin yet")
   end
 
   def flush(&block)
@@ -56,26 +46,10 @@ class LogStash::Codecs::JSONStream < LogStash::Codecs::Base
   end
 
   private
-
-  # from_json_parse uses the Event#from_json method to deserialize and directly produce events
-  def from_json_parse(json, &block)
-    LogStash::Event.from_json(json).each { |event| yield event }
-  rescue LogStash::Json::ParserError => e
-    @logger.warn("JSON parse error, original data now in message field", :error => e, :data => json)
-    yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"])
-  end
-
-  # legacy_parse uses the LogStash::Json class to deserialize json
-  def legacy_parse(json, &block)
-    # ignore empty/blank lines which LogStash::Json#load returns as nil
-    o = LogStash::Json.load(json)
-    yield(LogStash::Event.new(o)) if o
-  rescue LogStash::Json::ParserError => e
-    @logger.warn("JSON parse error, original data now in message field", :error => e, :data => json)
-    yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"])
+  def decode_unsafe(concatenated_json)
+    array_json = @converter.convert("[#{concatenated_json.gsub('}{', '},{')}]")
+    LogStash::Json.load(array_json).each do |decoded_event|
+       yield(LogStash::Event.new(decoded_event))
+    end
   end
-
-  # keep compatibility with all v2.x distributions. only in 2.3 will the Event#from_json method be introduced
-  # and we need to keep compatibility for all v2 releases.
-  alias_method :parse, LogStash::Event.respond_to?(:from_json) ? :from_json_parse : :legacy_parse
 end

data/logstash-codec-json_stream.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-json_stream'
-  s.version         = '0.0.2'
+  s.version         = '1.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads and writes non JSON Streams"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/json_stream_spec.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/json_stream"
+require "logstash/event"
+require "logstash/json"
+require "insist"
+
+describe LogStash::Codecs::JSONStream do
+
+  class LogStash::Codecs::JSONStream
+    public :decode_unsafe # use method without error logging for better visibility of errors
+  end
+
+  let(:codec_options) { {} }
+
+  context "default parser choice" do
+    subject do
+      LogStash::Codecs::JSONStream.new(codec_options)
+    end
+
+    it "should read multiple events" do
+      events = events_from_string(<<-EOS
+        {"messageType": "CONTROL_MESSAGE", "message": "foo"}
+        {"messageType": "DATA_MESSAGE", "logGroup": "testing", "logEvents": [
+          {"id": "4711", "@timestamp": "2018-06-18T13:36:25.484+00:00", "message": "{\\"tasks\\": \\"READING\\"}"},
+          {"id": "1848", "@timestamp": "1989-11-09T23:59:25.484+02:00", "message": "{\\"tasks\\": \\"WRITING\\"}"}
+        ]}
+EOS
+      )
+      insist { events.size } == 2
+
+      control_event = events[0]
+      data_event = events[1]
+
+      insist { control_event.is_a? LogStash::Event }
+      insist { control_event.get("messageType") } == "CONTROL_MESSAGE"
+      insist { control_event.get("message") } == "foo"
+
+      insist { data_event.is_a? LogStash::Event }
+      insist { data_event.get("messageType") } == "DATA_MESSAGE"
+      insist { data_event.get("logGroup") } == "testing"
+      insist { data_event.get("logEvents").size } == 2
+
+      insist { data_event.get("logEvents")[0]['id'] } == '4711'
+      insist { data_event.get("logEvents")[0]['@timestamp'] } == '2018-06-18T13:36:25.484+00:00'
+      insist { data_event.get("logEvents")[0]['message'] } == '{"tasks": "READING"}'
+
+      insist { data_event.get("logEvents")[1]['id'] } == '1848'
+      insist { data_event.get("logEvents")[1]['@timestamp'] } == '1989-11-09T23:59:25.484+02:00'
+      insist { data_event.get("logEvents")[1]['message'] } == '{"tasks": "WRITING"}'
+    end
+
+    it "should read multiple events from data" do
+      events = events_from_file('log-stream.valid-line-formatted')
+      insist { events.size } == 5
+
+      events.each do |event|
+        insist { event.is_a? LogStash::Event }
+        insist { event.get("logGroup") } == "test-core"
+        insist { event.get("messageType") } == "DATA_MESSAGE"
+        insist { event.get("logEvents").size } != 0
+        event.get("logEvents").each do |event|
+          insist { event["id"] } != nil
+          insist { event["message"] } != nil
+        end
+      end
+    end
+
+    it "should not fail with stacktrace" do
+      events = events_from_file('log-stream.minimal-failure-formatted')
+      insist { events.size } == 1
+
+      insist { events[0].is_a? LogStash::Event }
+      insist { events[0].get("logEvents").size } == 1
+      insist { events[0].get("logEvents")[0]['message']  =~ /Failed at: $\{springMacroRequestContext.getMessag\.\.\./ }
+    end
+  end
+
+  private
+  def events_from_file fixture_logfile_name
+    data = IO.read(File.join(File.dirname(__FILE__), "../../fixtures/#{fixture_logfile_name}"))
+    events_from_string data
+  end
+
+  def events_from_string data
+    events = []
+    data_without_formatting = data.gsub(/(\n|\s{2,})/, '')
+    subject.decode_unsafe(data_without_formatting) { |event| events << event }
+    events
+  end
+end

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-json_stream
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-23 00:00:00.000000000 Z
+date: 2018-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - <=
+    - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
@@ -24,16 +24,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - <=
+    - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
   name: logstash-codec-line
@@ -41,13 +41,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -55,10 +55,12 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: christian.herweg@gmail.com
 executables: []
 extensions: []
@@ -73,7 +75,7 @@ files:
 - docs/index.asciidoc
 - lib/logstash/codecs/json_stream.rb
 - logstash-codec-json_stream.gemspec
-- spec/codecs/json_lines_spec.rb
+- spec/codecs/json_stream_spec.rb
 homepage: https://github.com/cherweg/logstash-codec-json_stream
 licenses:
 - Apache License (2.0)
@@ -86,19 +88,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads and writes non JSON Streams
 test_files:
-- spec/codecs/json_lines_spec.rb
+- spec/codecs/json_stream_spec.rb

data/spec/codecs/json_lines_spec.rb

@@ -1,237 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/codecs/json_lines"
-require "logstash/event"
-require "logstash/json"
-require "insist"
-
-describe LogStash::Codecs::JSONLines do
-
-  let(:codec_options) { {} }
-
-  shared_examples :codec do
-
-  context "#decode" do
-    it "should return an event from json data" do
-      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
-      subject.decode(LogStash::Json.dump(data) + "\n") do |event|
-        insist { event.is_a? LogStash::Event }
-        insist { event.get("foo") } == data["foo"]
-        insist { event.get("baz") } == data["baz"]
-        insist { event.get("bah") } == data["bah"]
-      end
-    end
-
-    it "should return an event from json data when a newline is recieved" do
-      data = {"foo" => "bar", "baz" => {"bah" => ["a","b","c"]}}
-      subject.decode(LogStash::Json.dump(data)) do |event|
-        insist {false}
-      end
-      subject.decode("\n") do |event|
-        insist { event.is_a? LogStash::Event }
-        insist { event.get("foo") } == data["foo"]
-        insist { event.get("baz") } == data["baz"]
-        insist { event.get("bah") } == data["bah"]
-      end
-    end
-
-    context "when using custom delimiter" do
-      let(:delimiter) { "|" }
-      let(:line) { "{\"hey\":1}|{\"hey\":2}|{\"hey\":3}|" }
-      let(:codec_options) { { "delimiter" => delimiter } }
-
-      it "should decode multiple lines separated by the delimiter" do
-        result = []
-        subject.decode(line) { |event| result << event }
-        expect(result.size).to eq(3)
-        expect(result[0].get("hey")).to eq(1)
-        expect(result[1].get("hey")).to eq(2)
-        expect(result[2].get("hey")).to eq(3)
-      end
-    end
-
-    context "processing plain text" do
-      it "falls back to plain text" do
-        decoded = false
-        subject.decode("something that isn't json\n") do |event|
-          decoded = true
-          insist { event.is_a?(LogStash::Event) }
-          insist { event.get("message") } == "something that isn't json"
-          insist { event.get("tags") }.include?("_jsonparsefailure")
-        end
-        insist { decoded } == true
-      end
-    end
-
-    context "processing weird binary blobs" do
-      it "falls back to plain text and doesn't crash (LOGSTASH-1595)" do
-        decoded = false
-        blob = (128..255).to_a.pack("C*").force_encoding("ASCII-8BIT")
-        subject.decode(blob)
-        subject.decode("\n") do |event|
-          decoded = true
-          insist { event.is_a?(LogStash::Event) }
-          insist { event.get("message").encoding.to_s } == "UTF-8"
-        end
-        insist { decoded } == true
-      end
-    end
-
-    context "when json could not be parsed" do
-      let(:message)    { "random_message\n" }
-
-      it "add the failure tag" do
-        subject.decode(message) do |event|
-          expect(event).to include "tags"
-        end
-      end
-
-      it "uses an array to store the tags" do
-        subject.decode(message) do |event|
-          expect(event.get('tags')).to be_a Array
-        end
-      end
-
-      it "add a json parser failure tag" do
-        subject.decode(message) do |event|
-          expect(event.get('tags')).to include "_jsonparsefailure"
-        end
-      end
-    end
-
-    context "blank lines" do
-      let(:collector) { Array.new }
-
-      it "should ignore bare blanks" do
-        subject.decode("\n\n") do |event|
-          collector.push(event)
-        end
-        expect(collector.size).to eq(0)
-      end
-
-      it "should ignore in between blank lines" do
-        subject.decode("\n{\"a\":1}\n\n{\"b\":2}\n\n") do |event|
-          collector.push(event)
-        end
-        expect(collector.size).to eq(2)
-      end
-    end
-
-  end
-
-  context "#encode" do
-    let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}} }
-    let(:event) { LogStash::Event.new(data) }
-
-    it "should return json data" do
-      got_event = false
-      subject.on_event do |e, d|
-        insist { d } == "#{LogStash::Event.new(data).to_json}\n"
-        insist { LogStash::Json.load(d)["foo"] } == data["foo"]
-        insist { LogStash::Json.load(d)["baz"] } == data["baz"]
-        insist { LogStash::Json.load(d)["bah"] } == data["bah"]
-        got_event = true
-      end
-      subject.encode(event)
-      insist { got_event }
-    end
-
-    context "when using custom delimiter" do
-      let(:delimiter) { "|" }
-      let(:codec_options) { { "delimiter" => delimiter } }
-
-      it "should decode multiple lines separated by the delimiter" do
-        subject.on_event do |e, d|
-          insist { d } == "#{LogStash::Event.new(data).to_json}#{delimiter}"
-        end
-        subject.encode(event)
-      end
-    end
-  end
-
-  context 'reading from a simulated multiline json file without last newline' do
-    let(:input) do
-      %{{"field": "value1"}
-{"field": "value2"}}
-    end
-
-    let(:collector) { Array.new }
-
-    it 'should generate one event' do
-      subject.decode(input) do |event|
-        collector.push(event)
-      end
-      expect(collector.size).to eq(1)
-      expect(collector.first.get('field')).to eq('value1')
-    end
-  end
-
-  context 'reading from a simulated multiline json file with last newline' do
-    let(:input) do
-      %{{"field": "value1"}
-{"field": "value2"}
-}
-    end
-
-    let(:collector) { Array.new }
-
-    it 'should generate two events' do
-      subject.decode(input) do |event|
-        collector.push(event)
-      end
-      expect(collector.size).to eq(2)
-      expect(collector.first.get('field')).to eq('value1')
-      expect(collector.last.get('field')).to eq('value2')
-    end
-  end
-
-  end
-
-  context "forcing legacy parsing" do
-    it_behaves_like :codec do
-      subject do
-        # register method is called in the constructor
-        LogStash::Codecs::JSONLines.new(codec_options)
-      end
-
-      before(:each) do
-        # stub codec parse method to force use of the legacy parser.
-        # this is very implementation specific but I am not sure how
-        # this can be tested otherwise.
-        allow(subject).to receive(:parse) do |line, &block|
-          subject.send(:legacy_parse, line, &block)
-        end
-      end
-    end
-  end
-
-  context "default parser choice" do
-    # here we cannot force the use of the Event#from_json since if this test is run in the
-    # legacy context (no Java Event) it will fail but if in the new context, it will be picked up.
-    it_behaves_like :codec do
-      subject do
-        # register method is called in the constructor
-        LogStash::Codecs::JSONLines.new(codec_options)
-      end
-    end
-
-    context "flush" do
-      subject do
-        LogStash::Codecs::JSONLines.new(codec_options)
-      end
-
-      let(:input) { "{\"foo\":\"bar\"}" }
-
-      it "should flush buffered data'" do
-        result = []
-        subject.decode(input) { |e| result << e }
-        expect(result.size).to eq(0)
-
-        subject.flush { |e| result << e }
-        expect(result.size).to eq(1)
-
-        expect(result[0].get("foo")).to eq("bar")
-      end
-    end
-  end
-end