logstash-codec-json_lines 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa9392661527084579d1770404afdd4b22a81859
-  data.tar.gz: 16ca223e2f1db283f1b2079550e8afab340eda8b
+  metadata.gz: 4be023e15a5cf6663bc8bb2538583c6c8a8c306f
+  data.tar.gz: f9c0752674c58ff41cb8ea3f628782399f1dfbb4
 SHA512:
-  metadata.gz: f14010a1a30cfafcd7cffe1a20aee059d8baf9b5b70b902e7c2fefeac8342dca2053aa068913b9433b3b12834669305d60b5e58ed9b7d23b89d8c84d7a8ab4f3
-  data.tar.gz: eb86c593c9ac8c4bc523c391cbd0b0c65c7d23dc8f22f0716abeaf6f156702909bf1dbf3a7f463a560afb52b6fc1e95906d226d675e841e3b812ea3b0112beb9
+  metadata.gz: d7f37ba24a814fb4603c79c33ebc97fb5e05b29b32144cdd4da8f1f759d91b991844bbc22df5c2aa78e8deed600a7a9d77e776cca756d7cb194c532bdc4b4a09
+  data.tar.gz: c180c5b6db9f074c79336c79862ae56fd0d8d06cbe953e5ac1d240361e920d4f85c2be65ef708c98ec5934fa2df2580977436a545728e74653d4f00c12259340

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.1
+ - Improve documentation to warn about using this codec with a line oriented input.
+ - light refactor of decode method

data/CONTRIBUTORS

@@ -11,6 +11,7 @@ Contributors:
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
 * Tal Levy (talevy)
+* Guy Boertje (guyboertje)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/codecs/json_lines.rb

@@ -4,9 +4,12 @@ require "logstash/codecs/line"
 require "logstash/json"
 
 # This codec will decode streamed JSON that is newline delimited.
-# For decoding line-oriented JSON payload in the redis or file inputs,
-# for example, use the json codec instead.
 # Encoding will emit a single JSON string ending in a `\n`
+# NOTE: Do not use this codec if your source input is line-oriented JSON, for
+# example, redis or file inputs. Rather, use the json codec.
+# More info: This codec is expecting to receive a stream (string) of newline
+# terminated lines. The file input will produce a line string without a newline.
+# Therefore this codec cannot work with line oriented inputs.
 class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
   config_name "json_lines"
 
@@ -23,30 +26,33 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
   config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
 
   public
+
   def initialize(params={})
     super(params)
     @lines = LogStash::Codecs::Line.new
     @lines.charset = @charset
   end
 
-  public
   def decode(data)
-
     @lines.decode(data) do |event|
-      begin
-        yield LogStash::Event.new(LogStash::Json.load(event["message"]))
-      rescue LogStash::Json::ParserError => e
-        @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
-        yield LogStash::Event.new("message" => event["message"], "tags" => ["_jsonparsefailure"])
-      end
+      yield guard(event, data)
     end
   end # def decode
 
-  public
   def encode(event)
     # Tack on a \n for now because previously most of logstash's JSON
     # outputs emitted one per line, and whitespace is OK in json.
-    @on_event.call(event, event.to_json + NL)
+    @on_event.call(event, "#{event.to_json}#{NL}")
   end # def encode
 
-end # class LogStash::Codecs::JSON
+  private
+
+  def guard(event, data)
+    begin
+      LogStash::Event.new(LogStash::Json.load(event["message"]))
+    rescue LogStash::Json::ParserError => e
+      LogStash::Event.new("message" => event["message"], "tags" => ["_jsonparsefailure"])
+    end
+  end
+
+end # class LogStash::Codecs::JSONLines

data/logstash-codec-json_lines.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-json_lines'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This codec will decode streamed JSON that is newline delimited."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/json_lines_spec.rb

@@ -101,4 +101,40 @@ describe LogStash::Codecs::JSONLines do
       insist { got_event }
     end
   end
+
+  context 'reading from a simulated multiline json file without last newline' do
+    let(:input) do
+      %{{"field": "value1"}
+{"field": "value2"}}
+    end
+
+    let(:collector) { Array.new }
+
+    it 'should generate one event' do
+      subject.decode(input) do |event|
+        collector.push(event)
+      end
+      expect(collector.size).to eq(1)
+      expect(collector.first['field']).to eq('value1')
+    end
+  end
+
+  context 'reading from a simulated multiline json file with last newline' do
+    let(:input) do
+      %{{"field": "value1"}
+{"field": "value2"}
+}
+    end
+
+    let(:collector) { Array.new }
+
+    it 'should generate two events' do
+      subject.decode(input) do |event|
+        collector.push(event)
+      end
+      expect(collector.size).to eq(2)
+      expect(collector.first['field']).to eq('value1')
+      expect(collector.last['field']).to eq('value2')
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-json_lines
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -97,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: This codec will decode streamed JSON that is newline delimited.