logstash-filter-json 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f241f08d6c16daca28363d8e7ca71010d58ab19
-  data.tar.gz: 79e9fd1be3db90f64b5210c1ce58f17335d5be67
+  metadata.gz: 40d60f723d2d12a8b176d226b14d27e4f7b59adf
+  data.tar.gz: 09b010f8cd92b795c6af10eaf4b07ef908515768
 SHA512:
-  metadata.gz: 3e14beb6d325577ff3c5e08e006995b91197730fed4cd071df3dae23a80bae39d5d9d43156c5033dec44cd1b6bae89e48458f4c12e66bc01d9805346c74ec4c3
-  data.tar.gz: e552125bc5ac7090400171de2eb1aca84db47f104a6900ecd770802738916ece71b5815f9811ad22742afd83b9c4d7e6aa752f5c3cd8bbdbf847b6aece6ca8e9
+  metadata.gz: 9c2bc48511c1b4cae1f9c776479cde3b5558b9e7055971938e1329b5028af7ca6f645049216c4fa0dc0f43003f3ef458e79a6c46469ffe419c93a95882c37d3b
+  data.tar.gz: 4ad8488fb033dcd70f161e62f3a41f457650fbec31612dc177893af0867431f16936f3d564bf640ecad9e3f7a21f72896331ad7c5426e6862d7e4e4c9eaebc6a

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 2.0.3
+ - Refactored field references, better timestamp handling, code & specs cleanups
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-json-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-json-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/json.rb

@@ -46,54 +46,64 @@ class LogStash::Filters::Json < LogStash::Filters::Base
   # NOTE: if the `target` field already exists, it will be overwritten!
   config :target, :validate => :string
 
-  public
+  JSONPARSEFAILURE_TAG = "_jsonparsefailure"
+
   def register
     # Nothing to do here
-  end # def register
+  end
 
-  public
   def filter(event)
-    
-
-    @logger.debug("Running json filter", :event => event)
-
-    return unless event.include?(@source)
-
-    # TODO(colin) this field merging stuff below should be handled in Event.
+    @logger.debug? && @logger.debug("Running json filter", :event => event)
 
     source = event[@source]
+    return unless source
 
     begin
       parsed = LogStash::Json.load(source)
-      # If your parsed JSON is an array, we can't merge, so you must specify a
-      # destination to store the JSON, so you will get an exception about
-      if parsed.kind_of?(Array) && @target.nil?
-        raise('Parsed JSON arrays must have a destination in the configuration')
-      elsif @target.nil?
-        event.to_hash.merge! parsed
-      else
-        event[@target] = parsed
+    rescue => e
+      event.tag(JSONPARSEFAILURE_TAG)
+      @logger.warn("Error parsing json", :source => @source, :raw => source, :exception => e)
+      return
+    end
+
+    if @target
+      event[@target] = parsed
+    else
+      unless parsed.is_a?(Hash)
+        event.tag(JSONPARSEFAILURE_TAG)
+        @logger.warn("Parsed JSON object/hash requires a target configuration option", :source => @source, :raw => source)
+        return
       end
 
-      # If no target, we target the root of the event object. This can allow
-      # you to overwrite @timestamp and this will typically happen for json
-      # LogStash Event deserialized here.
-      if !@target && event.timestamp.is_a?(String)
-        event.timestamp = LogStash::Timestamp.parse_iso8601(event.timestamp)
+      # TODO: (colin) the timestamp initialization should be DRY'ed but exposing the similar code
+      # in the Event#init_timestamp method. See https://github.com/elastic/logstash/issues/4293
+
+      # a) since the parsed hash will be set in the event root, first extract any @timestamp field to properly initialized it
+      parsed_timestamp = parsed.delete(LogStash::Event::TIMESTAMP)
+      begin
+        timestamp = parsed_timestamp ? LogStash::Timestamp.coerce(parsed_timestamp) : nil
+      rescue LogStash::TimestampParserError => e
+        timestamp = nil
       end
 
-      filter_matched(event)
-    rescue => e
-      tag = "_jsonparsefailure"
-      event["tags"] ||= []
-      event["tags"] << tag unless event["tags"].include?(tag)
-      @logger.warn("Trouble parsing json", :source => @source,
-                   :raw => event[@source], :exception => e)
-      return
+      # b) then set all parsed fields in the event
+      parsed.each{|k, v| event[k] = v}
+
+      # c) finally re-inject proper @timestamp
+      if parsed_timestamp
+        if timestamp
+          event.timestamp = timestamp
+        else
+          event.timestamp = LogStash::Timestamp.new
+          @logger.warn("Unrecognized #{LogStash::Event::TIMESTAMP} value, setting current time to #{LogStash::Event::TIMESTAMP}, original in #{LogStash::Event::TIMESTAMP_FAILURE_FIELD} field", :value => parsed_timestamp.inspect)
+          event.tag(LogStash::Event::TIMESTAMP_FAILURE_TAG)
+          event[LogStash::Event::TIMESTAMP_FAILURE_FIELD] = parsed_timestamp.to_s
+        end
+      end
     end
 
-    @logger.debug("Event after json filter", :event => event)
-
-  end # def filter
+    filter_matched(event)
 
-end # class LogStash::Filters::Json
+    @logger.debug? && @logger.debug("Event after json filter", :event => event)
+  end
+end

data/logstash-filter-json.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-json'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This is a JSON parsing filter. It takes an existing field which contains JSON and expands it into an actual data structure within the Logstash event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/json_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/json"
 require "logstash/timestamp"
@@ -103,52 +105,72 @@ describe LogStash::Filters::Json do
     end
   end
 
-  context "when json could not be parsed" do
+  context "using message field source" do
 
     subject(:filter) {  LogStash::Filters::Json.new(config)  }
 
-    let(:message)    { "random_message" }
-    let(:config)     { {"source" => "message"} }
-    let(:event)      { LogStash::Event.new("message" => message) }
+    let(:config) { {"source" => "message"} }
+    let(:event) { LogStash::Event.new("message" => message) }
 
     before(:each) do
       filter.register
       filter.filter(event)
     end
 
-    it "add the failure tag" do
-      expect(event).to include "tags"
-    end
+    context "when json could not be parsed" do
+      let(:message) { "random_message" }
 
-    it "uses an array to store the tags" do
-      expect(event['tags']).to be_a Array
-    end
+      it "add the failure tag" do
+        expect(event).to include("tags")
+      end
 
-    it "add a json parser failure tag" do
-      expect(event['tags']).to include "_jsonparsefailure"
-    end
+      it "uses an array to store the tags" do
+        expect(event['tags']).to be_a(Array)
+      end
+
+      it "add a json parser failure tag" do
+        expect(event['tags']).to include("_jsonparsefailure")
+      end
 
-    context "there are two different errors added" do
+      context "there are two different errors added" do
 
-      let(:event)  { LogStash::Event.new("message" => message, "tags" => ["_anotherkinfoffailure"] ) }
+        let(:event)  { LogStash::Event.new("message" => message, "tags" => ["_anotherkinfoffailure"] ) }
 
-      it "pile the different error messages" do
-        expect(event['tags']).to include "_jsonparsefailure"
-      end
+        it "pile the different error messages" do
+          expect(event['tags']).to include("_jsonparsefailure")
+        end
 
-      it "keep the former error messages on the list" do
-        expect(event['tags']).to include "_anotherkinfoffailure"
+        it "keep the former error messages on the list" do
+          expect(event['tags']).to include("_anotherkinfoffailure")
+        end
       end
     end
 
     context "the JSON is an ArrayList" do
-
-      let(:message)  { "[1, 2, 3]" }
+      let(:message) { "[1, 2, 3]" }
 
       it "adds the failure tag" do
-        expect(event['tags']).to include "_jsonparsefailure"
+        expect(event['tags']).to include("_jsonparsefailure")
+      end
+    end
+
+    context "json contains valid timestamp" do
+      let(:message) { "{\"foo\":\"bar\", \"@timestamp\":\"2015-12-02T17:40:00.666Z\"}" }
+
+      it "should set json timestamp" do
+        expect(event.timestamp).to be_a(LogStash::Timestamp)
+        expect(event.timestamp.to_s).to eq("2015-12-02T17:40:00.666Z")
       end
     end
 
+    context "json contains invalid timestamp" do
+      let(:message) { "{\"foo\":\"bar\", \"@timestamp\":\"foobar\"}" }
+
+      it "should set timestamp to current time" do
+        expect(event.timestamp).to be_a(LogStash::Timestamp)
+        expect(event["tags"]).to include(LogStash::Event::TIMESTAMP_FAILURE_TAG)
+        expect(event[LogStash::Event::TIMESTAMP_FAILURE_FIELD]).to eq("foobar")
+      end
+    end
   end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-json
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,20 +28,22 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []