logstash-filter-yaml 0.1.1 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 068772ddb5786e465e7fcf687745e0fd388c96ee
4
- data.tar.gz: 07ed043a0d06d2730f139a7c4fd77c9fa81ee870
2
+ SHA256:
3
+ metadata.gz: 1624db57c6f4b7663f98c2e43c381afbad440024e42bb3502e36b2119d4a4cbe
4
+ data.tar.gz: 56beceb5113fc1b6633bfe9f0fab015708b1775fb139d1c82832c430c34ecb10
5
5
  SHA512:
6
- metadata.gz: 2cc42648efdfa4ec8eeda26c039469f63968ead2b85af3d672ac284b91a200ee7e6b767971db4daadd73fcaeb1fc79468612b70159ba46f379f450eb87a57178
7
- data.tar.gz: 63e8db5a84e53b99ea43b5c4943bd043d36cc16b93d804a3bf5e73a7c4ad789afdcb12b05ce45f4b457afb5510bf6ae2a981c6f7603f1afd8e2f4399b9ec99a6
6
+ metadata.gz: 9eab61b056b040d62e30e81ed58c5a0810f75f8d4c586109b10cd0b82264041d913f2a1cf1c9a7005a69183158c05300ccef1e887970c998f06137d82e27879f
7
+ data.tar.gz: 7a4e30e7e04330c82b77a6924fae6dae699f13f12142fb8609fce4b91cd1876298bace772c7c4473381ac252c64d9bb745854ddd86575c18423fdf98b298d1a6
@@ -1,5 +1,5 @@
1
- ## 2.0.0
2
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
3
- instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
4
- - Dependency on logstash-core update to 2.0
1
+ ## 1.0.0
2
+ - Added support for modern logstash versions
5
3
 
4
+ ## 0.1.1
5
+ - Initial version
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
1
+ Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@ START - GENERATED VARIABLES, DO NOT EDIT!
12
12
  END - GENERATED VARIABLES, DO NOT EDIT!
13
13
  ///////////////////////////////////////////
14
14
 
15
- [id="plugins-{type}-{plugin}"]
15
+ [id="plugins-{type}s-{plugin}"]
16
16
 
17
17
  === Yaml filter plugin
18
18
 
@@ -1,4 +1,5 @@
1
1
  # encoding: utf-8
2
+ require "logstash-core"
2
3
  require "logstash/filters/base"
3
4
  require "logstash/namespace"
4
5
  require "logstash/timestamp"
@@ -58,43 +59,52 @@ class LogStash::Filters::Yaml < LogStash::Filters::Base
58
59
 
59
60
  return unless event.include?(@source)
60
61
 
61
- source = event[@source]
62
+ source = event.get(@source)
62
63
 
63
- if @target.nil?
64
- # Default is to write to the root of the event.
65
- dest = event.to_hash
66
- else
67
- if @target == @source
68
- # Overwrite source
69
- event[@target] = {}
70
- else
71
- event[@target] ||= {}
72
- end
73
- dest = event[@target]
64
+ begin
65
+ parsed = YAML::load(source)
66
+ rescue => e
67
+ event.tag("_yamlparsefailure")
68
+ @logger.warn("Error parsing yaml", :source => @source,
69
+ :raw => event.get(@source), :exception => e.message)
70
+ return
74
71
  end
75
72
 
76
- begin
77
- dest.merge!(YAML::load(source))
73
+ if @target
74
+ event.set(@target, parsed)
75
+ else
76
+ unless parsed.is_a?(Hash)
77
+ event.tag("_yamlparsefailure")
78
+ @logger.warn("Parsed YAML object/hash requires a target configuration option", :source => @source, :raw => source)
79
+ return
80
+ end
78
81
 
79
- # If no target, we target the root of the event object. This can allow
80
- # you to overwrite @timestamp and this will typically happen for yaml
81
- # LogStash Event deserialized here.
82
- if !@target && event.timestamp.is_a?(String)
83
- event.timestamp = LogStash::Timestamp.parse_iso8601(event.timestamp)
82
+ # The following logic was copied from Json filter
83
+ # a) since the parsed hash will be set in the event root, first extract any @timestamp field to properly initialized it
84
+ parsed_timestamp = parsed.delete(LogStash::Event::TIMESTAMP)
85
+ begin
86
+ timestamp = parsed_timestamp ? LogStash::Timestamp.coerce(parsed_timestamp) : nil
87
+ rescue LogStash::TimestampParserError => e
88
+ timestamp = nil
84
89
  end
85
90
 
86
- filter_matched(event)
87
- rescue => e
88
- tag = "_yamlparsefailure"
89
- event["tags"] ||= []
90
- event["tags"] << tag unless event["tags"].include?(tag)
91
- @logger.warn("Trouble parsing yaml", :source => @source,
92
- :raw => event[@source], :exception => e)
93
- return
91
+ # b) then set all parsed fields in the event
92
+ parsed.each{|k, v| event.set(k, v)}
93
+
94
+ # c) finally re-inject proper @timestamp
95
+ if parsed_timestamp
96
+ if timestamp
97
+ event.timestamp = timestamp
98
+ else
99
+ event.timestamp = LogStash::Timestamp.new
100
+ @logger.warn("Unrecognized #{LogStash::Event::TIMESTAMP} value, setting current time to #{LogStash::Event::TIMESTAMP}, original in #{LogStash::Event::TIMESTAMP_FAILURE_FIELD} field", :value => parsed_timestamp.inspect)
101
+ event.tag(LogStash::Event::TIMESTAMP_FAILURE_TAG)
102
+ event.set(LogStash::Event::TIMESTAMP_FAILURE_FIELD, parsed_timestamp.to_s)
103
+ end
104
+ end
94
105
  end
95
106
 
96
- @logger.debug("Event after yaml filter", :event => event)
97
-
107
+ filter_matched(event)
108
+ @logger.debug? && @logger.debug("Event after yaml filter", :event => event.inspect)
98
109
  end # def filter
99
-
100
110
  end # class LogStash::Filters::Yaml
@@ -1,11 +1,11 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-yaml'
4
- s.version = '0.1.1'
4
+ s.version = '1.0.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This is a YAML parsing filter. It takes an existing field which contains YAML and expands it into an actual data structure within the Logstash event."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
8
- s.authors = ["Jean-Philippe Briend"]
8
+ s.authors = ["Elastic", "Jean-Philippe Briend"]
9
9
  s.email = 'jeanphilippe.briend@gmail.com'
10
10
  s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
11
11
  s.require_paths = ["lib"]
@@ -20,8 +20,7 @@ Gem::Specification.new do |s|
20
20
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
21
21
 
22
22
  # Gem dependencies
23
- s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
23
+ s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
24
24
 
25
25
  s.add_development_dependency 'logstash-devutils'
26
26
  end
27
-
@@ -1,89 +1,66 @@
1
+ require "logstash-core"
1
2
  require "logstash/devutils/rspec/spec_helper"
2
3
  require "logstash/filters/yaml"
3
4
  require "logstash/timestamp"
4
5
 
5
6
  describe LogStash::Filters::Yaml do
7
+ subject { described_class.new(config) }
8
+ let(:event) { LogStash::Event.new(data) }
6
9
 
7
10
  describe "parse message into the event" do
8
- config <<-CONFIG
9
- filter {
10
- yaml {
11
- # Parse message as YAML
12
- source => "message"
13
- }
14
- }
15
- CONFIG
11
+ let(:config) { { "source" => "message" } }
12
+ let(:data) { { "message" => '{ hello: "world", list: [ 1, 2, 3 ], hash: { k: v }, sometime: "2013-10-19T00:14:32.996Z" }' } }
16
13
 
17
- sample '{ hello: "world", list: [ 1, 2, 3 ], hash: { k: v }, sometime: "2013-10-19T00:14:32.996Z" }' do
18
- insist { subject["hello"] } == "world"
19
- insist { subject["list" ].to_a } == [1,2,3] # to_a for JRuby + JrJacksom which creates Java ArrayList
20
- insist { subject["hash"] } == { "k" => "v" }
14
+ it "parses correctly" do
15
+ subject.filter(event)
16
+ expect(event.get("hello")).to eq("world")
17
+ expect(event.get("list" ).to_a).to eq([1,2,3]) # to_a for JRuby + JrJacksom which creates Java ArrayList
18
+ expect(event.get("hash")).to eq({ "k" => "v" })
21
19
  end
22
20
  end
23
21
 
24
22
  describe "parse message into a target field" do
25
- config <<-CONFIG
26
- filter {
27
- yaml {
28
- # Parse message as YAML, store the results in the 'data' field'
29
- source => "message"
30
- target => "data"
31
- }
32
- }
33
- CONFIG
23
+ let(:config) { { "source" => "message", "target" => "data" } }
24
+ let(:data) { { "message" => '{ hello: "world", list: [ 1, 2, 3 ], "hash": { k: v } }' } }
34
25
 
35
- sample '{ hello: "world", list: [ 1, 2, 3 ], "hash": { k: v } }' do
36
- insist { subject["data"]["hello"] } == "world"
37
- insist { subject["data"]["list" ].to_a } == [1,2,3] # to_a for JRuby + JrJacksom which creates Java ArrayList
38
- insist { subject["data"]["hash"] } == { "k" => "v" }
26
+ it "parses correctly" do
27
+ subject.filter(event)
28
+ expect(event.get("data")["hello"]).to eq("world")
29
+ expect(event.get("data")["list" ].to_a).to eq([1,2,3]) # to_a for JRuby + JrJacksom which creates Java ArrayList
30
+ expect(event.get("data")["hash"]).to eq({ "k" => "v" })
39
31
  end
40
32
  end
41
33
 
42
34
  describe "tag invalid yaml" do
43
- config <<-CONFIG
44
- filter {
45
- yaml {
46
- # Parse message as YAML, store the results in the 'data' field'
47
- source => "message"
48
- target => "data"
49
- }
50
- }
51
- CONFIG
35
+ # Parse message as YAML, store the results in the 'data' field'
36
+ let(:config) { { "source" => "message", "target" => "data" } }
37
+ let(:data) { { "message" => "'" } }
52
38
 
53
- sample "invalid yaml" do
54
- insist { subject["tags"] }.include?("_yamlparsefailure")
39
+ it "tags the failure to parse" do
40
+ subject.filter(event)
41
+ expect(event.get("tags")).to include("_yamlparsefailure")
55
42
  end
56
43
  end
57
44
 
58
45
  describe "testing @timestamp" do
59
- config <<-CONFIG
60
- filter {
61
- yaml {
62
- source => "message"
63
- }
64
- }
65
- CONFIG
66
-
67
- sample "{ \"@timestamp\": \"2013-10-19T00:14:32.996Z\" }" do
68
- insist { subject["@timestamp"] }.is_a?(LogStash::Timestamp)
69
- insist { YAML::dump(subject["@timestamp"]) } == "--- !ruby/object:LogStash::Timestamp\ntime: 2013-10-19 00:14:32.996000000 Z\n"
46
+ let(:config) { { "source" => "message" } }
47
+ let(:date) { "2013-10-19T00:14:32.996Z" }
48
+ let(:data) { { "message" => "{ \"@timestamp\": \"#{date}\" }" } }
49
+ it "parses correctly" do
50
+ subject.filter(event)
51
+ expect(event.timestamp).to be_a(LogStash::Timestamp)
52
+ expect(event.timestamp.to_s).to eq(date)
70
53
  end
71
54
  end
72
55
 
73
56
  describe "source == target" do
74
- config <<-CONFIG
75
- filter {
76
- yaml {
77
- source => "example"
78
- target => "example"
79
- }
80
- }
81
- CONFIG
57
+ let(:config) { { "source" => "example", "target" => "example" } }
58
+ let(:data) { { "example" => "{ hello: world }" } }
82
59
 
83
- sample({ "example" => "{ hello: world }" }) do
84
- insist { subject["example"] }.is_a?(Hash)
85
- insist { subject["example"]["hello"] } == "world"
60
+ it "parses correctly" do
61
+ subject.filter(event)
62
+ expect(event.get("example")).to be_a(Hash)
63
+ expect(event.get("example")["hello"]).to eq("world")
86
64
  end
87
65
  end
88
-
89
66
  end
metadata CHANGED
@@ -1,35 +1,30 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-yaml
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
+ - Elastic
7
8
  - Jean-Philippe Briend
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2017-06-23 00:00:00.000000000 Z
12
+ date: 2018-11-16 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  requirement: !ruby/object:Gem::Requirement
15
16
  requirements:
16
- - - ">="
17
- - !ruby/object:Gem::Version
18
- version: 1.4.0
19
- - - "<"
17
+ - - "~>"
20
18
  - !ruby/object:Gem::Version
21
- version: 2.0.0
22
- name: logstash-core
19
+ version: '2.0'
20
+ name: logstash-core-plugin-api
23
21
  prerelease: false
24
22
  type: :runtime
25
23
  version_requirements: !ruby/object:Gem::Requirement
26
24
  requirements:
27
- - - ">="
28
- - !ruby/object:Gem::Version
29
- version: 1.4.0
30
- - - "<"
25
+ - - "~>"
31
26
  - !ruby/object:Gem::Version
32
- version: 2.0.0
27
+ version: '2.0'
33
28
  - !ruby/object:Gem::Dependency
34
29
  requirement: !ruby/object:Gem::Requirement
35
30
  requirements:
@@ -44,7 +39,9 @@ dependencies:
44
39
  - - ">="
45
40
  - !ruby/object:Gem::Version
46
41
  version: '0'
47
- description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
42
+ description: This gem is a logstash plugin required to be installed on top of the
43
+ Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
44
+ a stand-alone program
48
45
  email: jeanphilippe.briend@gmail.com
49
46
  executables: []
50
47
  extensions: []
@@ -82,9 +79,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
82
79
  version: '0'
83
80
  requirements: []
84
81
  rubyforge_project:
85
- rubygems_version: 2.4.8
82
+ rubygems_version: 2.6.13
86
83
  signing_key:
87
84
  specification_version: 4
88
- summary: This is a YAML parsing filter. It takes an existing field which contains YAML and expands it into an actual data structure within the Logstash event.
85
+ summary: This is a YAML parsing filter. It takes an existing field which contains
86
+ YAML and expands it into an actual data structure within the Logstash event.
89
87
  test_files:
90
88
  - spec/filters/yaml_spec.rb