logstash-filter-yaml 0.1.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 068772ddb5786e465e7fcf687745e0fd388c96ee
-  data.tar.gz: 07ed043a0d06d2730f139a7c4fd77c9fa81ee870
+SHA256:
+  metadata.gz: 1624db57c6f4b7663f98c2e43c381afbad440024e42bb3502e36b2119d4a4cbe
+  data.tar.gz: 56beceb5113fc1b6633bfe9f0fab015708b1775fb139d1c82832c430c34ecb10
 SHA512:
-  metadata.gz: 2cc42648efdfa4ec8eeda26c039469f63968ead2b85af3d672ac284b91a200ee7e6b767971db4daadd73fcaeb1fc79468612b70159ba46f379f450eb87a57178
-  data.tar.gz: 63e8db5a84e53b99ea43b5c4943bd043d36cc16b93d804a3bf5e73a7c4ad789afdcb12b05ce45f4b457afb5510bf6ae2a981c6f7603f1afd8e2f4399b9ec99a6
+  metadata.gz: 9eab61b056b040d62e30e81ed58c5a0810f75f8d4c586109b10cd0b82264041d913f2a1cf1c9a7005a69183158c05300ccef1e887970c998f06137d82e27879f
+  data.tar.gz: 7a4e30e7e04330c82b77a6924fae6dae699f13f12142fb8609fce4b91cd1876298bace772c7c4473381ac252c64d9bb745854ddd86575c18423fdf98b298d1a6

data/CHANGELOG.md

@@ -1,5 +1,5 @@
-## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
-   instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
- - Dependency on logstash-core update to 2.0
+## 1.0.0
+  - Added support for modern logstash versions
 
+## 0.1.1
+  - Initial version

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -12,7 +12,7 @@ START - GENERATED VARIABLES, DO NOT EDIT!
 END - GENERATED VARIABLES, DO NOT EDIT!
 ///////////////////////////////////////////
 
-[id="plugins-{type}-{plugin}"]
+[id="plugins-{type}s-{plugin}"]
 
 === Yaml filter plugin
 

data/lib/logstash/filters/yaml.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require "logstash-core"
 require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/timestamp"
@@ -58,43 +59,52 @@ class LogStash::Filters::Yaml < LogStash::Filters::Base
 
     return unless event.include?(@source)
 
-    source = event[@source]
+    source = event.get(@source)
 
-    if @target.nil?
-      # Default is to write to the root of the event.
-      dest = event.to_hash
-    else
-      if @target == @source
-        # Overwrite source
-        event[@target] = {}
-      else
-        event[@target] ||= {}
-      end
-      dest = event[@target]
+    begin
+      parsed = YAML::load(source)
+    rescue => e
+      event.tag("_yamlparsefailure")
+      @logger.warn("Error parsing yaml", :source => @source,
+                   :raw => event.get(@source), :exception => e.message)
+      return
     end
 
-    begin
-      dest.merge!(YAML::load(source))
+    if @target
+      event.set(@target, parsed)
+    else
+      unless parsed.is_a?(Hash)
+        event.tag("_yamlparsefailure")
+        @logger.warn("Parsed YAML object/hash requires a target configuration option", :source => @source, :raw => source)
+        return
+      end
 
-      # If no target, we target the root of the event object. This can allow
-      # you to overwrite @timestamp and this will typically happen for yaml
-      # LogStash Event deserialized here.
-      if !@target && event.timestamp.is_a?(String)
-        event.timestamp = LogStash::Timestamp.parse_iso8601(event.timestamp)
+      # The following logic was copied from Json filter
+      # a) since the parsed hash will be set in the event root, first extract any @timestamp field to properly initialized it
+      parsed_timestamp = parsed.delete(LogStash::Event::TIMESTAMP)
+      begin
+        timestamp = parsed_timestamp ? LogStash::Timestamp.coerce(parsed_timestamp) : nil
+      rescue LogStash::TimestampParserError => e
+        timestamp = nil
       end
 
-      filter_matched(event)
-    rescue => e
-      tag = "_yamlparsefailure"
-      event["tags"] ||= []
-      event["tags"] << tag unless event["tags"].include?(tag)
-      @logger.warn("Trouble parsing yaml", :source => @source,
-                   :raw => event[@source], :exception => e)
-      return
+      # b) then set all parsed fields in the event
+      parsed.each{|k, v| event.set(k, v)}
+
+      # c) finally re-inject proper @timestamp
+      if parsed_timestamp
+        if timestamp
+          event.timestamp = timestamp
+        else
+          event.timestamp = LogStash::Timestamp.new
+          @logger.warn("Unrecognized #{LogStash::Event::TIMESTAMP} value, setting current time to #{LogStash::Event::TIMESTAMP}, original in #{LogStash::Event::TIMESTAMP_FAILURE_FIELD} field", :value => parsed_timestamp.inspect)
+          event.tag(LogStash::Event::TIMESTAMP_FAILURE_TAG)
+          event.set(LogStash::Event::TIMESTAMP_FAILURE_FIELD, parsed_timestamp.to_s)
+        end
+      end
     end
 
-    @logger.debug("Event after yaml filter", :event => event)
-
+    filter_matched(event)
+    @logger.debug? && @logger.debug("Event after yaml filter", :event => event.inspect)
   end # def filter
-
 end # class LogStash::Filters::Yaml

data/logstash-filter-yaml.gemspec

@@ -1,11 +1,11 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-yaml'
-  s.version         = '0.1.1'
+  s.version         = '1.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This is a YAML parsing filter. It takes an existing field which contains YAML and expands it into an actual data structure within the Logstash event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors         = ["Jean-Philippe Briend"]
+  s.authors         = ["Elastic", "Jean-Philippe Briend"]
   s.email           = 'jeanphilippe.briend@gmail.com'
   s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
   s.require_paths = ["lib"]
@@ -20,8 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
 
   s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/filters/yaml_spec.rb

@@ -1,89 +1,66 @@
+require "logstash-core"
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/yaml"
 require "logstash/timestamp"
 
 describe LogStash::Filters::Yaml do
+  subject { described_class.new(config) }
+  let(:event) { LogStash::Event.new(data) }
 
   describe "parse message into the event" do
-    config <<-CONFIG
-      filter {
-        yaml {
-          # Parse message as YAML
-          source => "message"
-        }
-      }
-    CONFIG
+    let(:config) { { "source" => "message" } }
+    let(:data) { { "message" => '{ hello: "world", list: [ 1, 2, 3 ], hash: { k: v }, sometime: "2013-10-19T00:14:32.996Z" }' } }
 
-    sample '{ hello: "world", list: [ 1, 2, 3 ], hash: { k: v }, sometime: "2013-10-19T00:14:32.996Z" }' do
-      insist { subject["hello"] } == "world"
-      insist { subject["list" ].to_a } == [1,2,3] # to_a for JRuby + JrJacksom which creates Java ArrayList
-      insist { subject["hash"] } == { "k" => "v" }
+    it "parses correctly" do
+      subject.filter(event)
+      expect(event.get("hello")).to eq("world")
+      expect(event.get("list" ).to_a).to eq([1,2,3]) # to_a for JRuby + JrJacksom which creates Java ArrayList
+      expect(event.get("hash")).to eq({ "k" => "v" })
     end
   end
 
   describe "parse message into a target field" do
-    config <<-CONFIG
-      filter {
-        yaml {
-          # Parse message as YAML, store the results in the 'data' field'
-          source => "message"
-          target => "data"
-        }
-      }
-    CONFIG
+    let(:config) { { "source" => "message", "target" => "data" } }
+    let(:data) { { "message" => '{ hello: "world", list: [ 1, 2, 3 ], "hash": { k: v } }' } }
 
-    sample '{ hello: "world", list: [ 1, 2, 3 ], "hash": { k: v } }' do
-      insist { subject["data"]["hello"] } == "world"
-      insist { subject["data"]["list" ].to_a } == [1,2,3] # to_a for JRuby + JrJacksom which creates Java ArrayList
-      insist { subject["data"]["hash"] } == { "k" => "v" }
+    it "parses correctly" do
+      subject.filter(event)
+      expect(event.get("data")["hello"]).to eq("world")
+      expect(event.get("data")["list" ].to_a).to eq([1,2,3]) # to_a for JRuby + JrJacksom which creates Java ArrayList
+      expect(event.get("data")["hash"]).to eq({ "k" => "v" })
     end
   end
 
   describe "tag invalid yaml" do
-    config <<-CONFIG
-      filter {
-        yaml {
-          # Parse message as YAML, store the results in the 'data' field'
-          source => "message"
-          target => "data"
-        }
-      }
-    CONFIG
+    # Parse message as YAML, store the results in the 'data' field'
+    let(:config) { { "source" => "message", "target" => "data" } }
+    let(:data) { { "message" => "'" } }
 
-    sample "invalid yaml" do
-      insist { subject["tags"] }.include?("_yamlparsefailure")
+    it "tags the failure to parse" do
+      subject.filter(event)
+      expect(event.get("tags")).to include("_yamlparsefailure")
     end
   end
 
   describe "testing @timestamp" do
-    config <<-CONFIG
-      filter {
-        yaml {
-          source => "message"
-        }
-      }
-    CONFIG
-
-    sample "{ \"@timestamp\": \"2013-10-19T00:14:32.996Z\" }" do
-      insist { subject["@timestamp"] }.is_a?(LogStash::Timestamp)
-      insist { YAML::dump(subject["@timestamp"]) } == "--- !ruby/object:LogStash::Timestamp\ntime: 2013-10-19 00:14:32.996000000 Z\n"
+    let(:config) { { "source" => "message" } }
+    let(:date) { "2013-10-19T00:14:32.996Z" }
+    let(:data) { { "message" => "{ \"@timestamp\": \"#{date}\" }"  } }
+    it "parses correctly" do
+      subject.filter(event)
+      expect(event.timestamp).to be_a(LogStash::Timestamp)
+      expect(event.timestamp.to_s).to eq(date)
     end
   end
 
   describe "source == target" do
-    config <<-CONFIG
-      filter {
-        yaml {
-          source => "example"
-          target => "example"
-        }
-      }
-    CONFIG
+    let(:config) { { "source" => "example", "target" => "example" } }
+    let(:data) { { "example" => "{ hello: world }" } }
 
-    sample({ "example" => "{ hello: world }" }) do
-      insist { subject["example"] }.is_a?(Hash)
-      insist { subject["example"]["hello"] } == "world"
+    it "parses correctly" do
+      subject.filter(event)
+      expect(event.get("example")).to be_a(Hash)
+      expect(event.get("example")["hello"]).to eq("world")
     end
   end
-
 end

metadata

@@ -1,35 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-yaml
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
+- Elastic
 - Jean-Philippe Briend
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-23 00:00:00.000000000 Z
+date: 2018-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
-  name: logstash-core
+        version: '2.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -44,7 +39,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
 email: jeanphilippe.briend@gmail.com
 executables: []
 extensions: []
@@ -82,9 +79,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
-summary: This is a YAML parsing filter. It takes an existing field which contains YAML and expands it into an actual data structure within the Logstash event.
+summary: This is a YAML parsing filter. It takes an existing field which contains
+  YAML and expands it into an actual data structure within the Logstash event.
 test_files:
 - spec/filters/yaml_spec.rb