logstash-filter-collect 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/CONTRIBUTORS +10 -0
- data/DEVELOPER.md +2 -0
- data/Gemfile +3 -0
- data/LICENSE +11 -0
- data/README.md +44 -0
- data/lib/logstash/filters/collect.rb +68 -0
- data/logstash-filter-collect.gemspec +22 -0
- data/spec/filters/collect_spec.rb +71 -0
- data/spec/spec_helper.rb +3 -0
- metadata +85 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 8bba7ec257f540842aa9d9f509d52ad982401ae6
|
4
|
+
data.tar.gz: a14a1f5b055e437c0d769123cfe586159485c542
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c9181a4cfa77a6dc954e9b90e0b05e12e39ff4d636b249c487a4a341cd420fd3eb80c40a450b56e9513e4fc7e101f50307cecb0ec90e797ae2abb81ddeefd9e2
|
7
|
+
data.tar.gz: 61ceb2a840cba9c529f24136197143ac88f2889cda9156fcc736d36ce8b31445db2ffc73a89c52406c016237aa8419104f1cd63233a1eba4b555230b42ccc3ea
|
data/CHANGELOG.md
ADDED
data/CONTRIBUTORS
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
The following is a list of people who have contributed ideas, code, bug
|
2
|
+
reports, or in general have helped logstash along its way.
|
3
|
+
|
4
|
+
Contributors:
|
5
|
+
* Mike Baranski - mike.baranski@gmail.com
|
6
|
+
|
7
|
+
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
8
|
+
Logstash, and you aren't on the list above and want to be, please let us know
|
9
|
+
and we'll make sure you're here. Contributions from folks like you are what make
|
10
|
+
open source awesome.
|
data/DEVELOPER.md
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
2
|
+
you may not use this file except in compliance with the License.
|
3
|
+
You may obtain a copy of the License at
|
4
|
+
|
5
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
6
|
+
|
7
|
+
Unless required by applicable law or agreed to in writing, software
|
8
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
9
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
10
|
+
See the License for the specific language governing permissions and
|
11
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# Logstash Collect Plugin
|
2
|
+
|
3
|
+
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
|
+
|
5
|
+
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
6
|
+
|
7
|
+
Author: Mike Baranski (mike.baranski@gmail.com). Contributions are welcome.
|
8
|
+
|
9
|
+
## License ##
|
10
|
+
|
11
|
+
Copyright (c) 2014–2017 Mike Baranski <http://www.mikeski.net>
|
12
|
+
|
13
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
14
|
+
you may not use this file except in compliance with the License.
|
15
|
+
You may obtain a copy of the License at
|
16
|
+
|
17
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
18
|
+
|
19
|
+
Unless required by applicable law or agreed to in writing, software
|
20
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
21
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
22
|
+
See the License for the specific language governing permissions and
|
23
|
+
limitations under the License.
|
24
|
+
|
25
|
+
## Installation
|
26
|
+
|
27
|
+
## Usage
|
28
|
+
|
29
|
+
This plugin takes a list of objects in your event and turns it into a
|
30
|
+
list of strings. The original purpose of this was to take something like this:
|
31
|
+
|
32
|
+
lop => [{ 'person' => {'id' => 12, 'name' => 'Mike'} }, {'person' => {'id' => 13, 'name' => 'Sam'}}]
|
33
|
+
|
34
|
+
and turn it into a list like this:
|
35
|
+
|
36
|
+
names => ['Mike', 'Sam']
|
37
|
+
|
38
|
+
With this filter plugin, the following configuration will do that:
|
39
|
+
|
40
|
+
collect => {
|
41
|
+
'field' => 'people',
|
42
|
+
'property' => ['person' 'name'],
|
43
|
+
'collection' => 'names'
|
44
|
+
}
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/filters/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
|
5
|
+
# This filter will replace the contents of the default
|
6
|
+
# message field with whatever you specify in the configuration.
|
7
|
+
#
|
8
|
+
# It is only intended to be used as an .
|
9
|
+
class LogStash::Filters::Collect < LogStash::Filters::Base
|
10
|
+
|
11
|
+
# This is how you
|
12
|
+
# configure this filter from your Logstash config.
|
13
|
+
#
|
14
|
+
# filter {
|
15
|
+
# collect {
|
16
|
+
# field => 'object_array_field',
|
17
|
+
# property => ['property', 'of', 'each', 'object'],
|
18
|
+
# collection => 'output_list_name',
|
19
|
+
# }
|
20
|
+
# }
|
21
|
+
#
|
22
|
+
config_name 'collect'
|
23
|
+
|
24
|
+
# Replace the message with this value.
|
25
|
+
config :field, :validate => :string, :required => true
|
26
|
+
config :property, :validate => :array, :required => true
|
27
|
+
config :collection, :validate => :string, :required => true, :default => 'collection'
|
28
|
+
config :error_tags, :validate => :array, :required => true, :default => ['_collect_error']
|
29
|
+
|
30
|
+
public
|
31
|
+
def register
|
32
|
+
# Add instance variables
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
def plugin_error(message, event)
|
37
|
+
@logger.error("Collect filter error: #{message}")
|
38
|
+
LogStash::Util::Decorators.add_tags(@error_tags, event, "filters/#{self.class.name}")
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
def get_nested_property_from_object(property_array, source)
|
43
|
+
interim_object = source
|
44
|
+
@property.each { |x|
|
45
|
+
interim_object = interim_object[x]
|
46
|
+
}
|
47
|
+
return interim_object
|
48
|
+
end
|
49
|
+
|
50
|
+
public
|
51
|
+
def filter(event)
|
52
|
+
object_array = event.get(@field)
|
53
|
+
if not object_array.class == Array
|
54
|
+
plugin_error("Error, #{@field} is not an array", event)
|
55
|
+
return
|
56
|
+
end
|
57
|
+
|
58
|
+
new_collection = []
|
59
|
+
object_array.each { |obj|
|
60
|
+
# Add the value to the new collection
|
61
|
+
new_collection.push(get_nested_property_from_object(@property, obj))
|
62
|
+
}
|
63
|
+
|
64
|
+
event.set(@collection, new_collection)
|
65
|
+
filter_matched(event)
|
66
|
+
end # def filter
|
67
|
+
|
68
|
+
end # class LogStash::Filters::Collect
|
@@ -0,0 +1,22 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-filter-collect'
|
3
|
+
s.version = '1.0.0'
|
4
|
+
s.licenses = ['Apache License (2.0)']
|
5
|
+
s.summary = 'Collect list of properties from list of objects'
|
6
|
+
s.homepage = 'http://mikeski.net'
|
7
|
+
s.authors = ['Mike Baranski']
|
8
|
+
s.email = 'mike.baranski@gmail.com'
|
9
|
+
s.require_paths = ['lib']
|
10
|
+
|
11
|
+
# Files
|
12
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
13
|
+
# Tests
|
14
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
15
|
+
|
16
|
+
# Special flag to let us know this is actually a logstash plugin
|
17
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
18
|
+
|
19
|
+
# Gem dependencies
|
20
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
21
|
+
s.add_development_dependency 'logstash-devutils'
|
22
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative '../spec_helper'
|
3
|
+
require "logstash/filters/collect"
|
4
|
+
|
5
|
+
describe LogStash::Filters::Collect do
|
6
|
+
#config.expect_with(:rspec) { |c| c.syntax = :should }
|
7
|
+
|
8
|
+
def get_event(contents = {})
|
9
|
+
contents['@timestamp'] = LogStash::Timestamp.new
|
10
|
+
contents['host'] = 'somehost'
|
11
|
+
contents['message'] = 'some message for event'
|
12
|
+
LogStash::Event.new(contents)
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'Test get_event method' do
|
16
|
+
e = get_event({'field1': 'val1'})
|
17
|
+
e.set('test', 1234)
|
18
|
+
e.to_hash
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'Test new filter no config' do
|
22
|
+
expect { LogStash::Filters::Collect.new({}) }.to raise_error(LogStash::ConfigurationError)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'Property can be a list' do
|
26
|
+
config = {'field' => 'object_array', 'property' => %w(property to get)}
|
27
|
+
LogStash::Filters::Collect.new(config)
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'Property can be a single string' do
|
31
|
+
config = {'field' => 'object_array', 'property' => 'prop'}
|
32
|
+
LogStash::Filters::Collect.new(config)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'Override collection default' do
|
36
|
+
config = {'field' => 'object_array', 'property' => 'prop', 'collection' => 'some_collection'}
|
37
|
+
LogStash::Filters::Collect.new(config)
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'Passing in invalid collection adds error tag' do
|
41
|
+
config = {'field' => 'people', 'property' => 'name', 'collection' => 'names'}
|
42
|
+
f = LogStash::Filters::Collect.new(config)
|
43
|
+
e = get_event({'people' => 'scalar value'})
|
44
|
+
f.filter(e)
|
45
|
+
e.get('tags').should include('_collect_error')
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'Test list of simple objects (only 1 level to de-reference)' do
|
49
|
+
ppl = [{'id' => 12, 'name' => 'Mike'}, {'id' => 13, 'name' => 'Sam'}]
|
50
|
+
config = {'field' => 'people', 'property' => 'name', 'collection' => 'names'}
|
51
|
+
f = LogStash::Filters::Collect.new(config)
|
52
|
+
e = get_event({'people' => ppl})
|
53
|
+
f.filter(e)
|
54
|
+
|
55
|
+
expect(e.get('names').class).to be(Array)
|
56
|
+
expect(e.get('names').include?('Mike')).to be(true)
|
57
|
+
expect(e.get('names').include?('Sam')).to be(true)
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'Test list of not-so-simple objects (> 1 level to de-reference)' do
|
61
|
+
ppl = [{ 'person' => {'id' => 12, 'name' => 'Mike'} }, {'person' => {'id' => 13, 'name' => 'Sam'}}]
|
62
|
+
config = {'field' => 'people', 'property' => %w(person name), 'collection' => 'names'}
|
63
|
+
f = LogStash::Filters::Collect.new(config)
|
64
|
+
e = get_event({'people' => ppl})
|
65
|
+
f.filter(e)
|
66
|
+
expect(e.get('names').class).to be(Array)
|
67
|
+
expect(e.get('names').include?('Mike')).to be(true)
|
68
|
+
expect(e.get('names').include?('Sam')).to be(true)
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-filter-collect
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Mike Baranski
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-01-09 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - "~>"
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '2.0'
|
19
|
+
name: logstash-core-plugin-api
|
20
|
+
prerelease: false
|
21
|
+
type: :runtime
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
name: logstash-devutils
|
34
|
+
prerelease: false
|
35
|
+
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
description:
|
42
|
+
email: mike.baranski@gmail.com
|
43
|
+
executables: []
|
44
|
+
extensions: []
|
45
|
+
extra_rdoc_files: []
|
46
|
+
files:
|
47
|
+
- CHANGELOG.md
|
48
|
+
- CONTRIBUTORS
|
49
|
+
- DEVELOPER.md
|
50
|
+
- Gemfile
|
51
|
+
- LICENSE
|
52
|
+
- README.md
|
53
|
+
- lib/logstash/filters/collect.rb
|
54
|
+
- logstash-filter-collect.gemspec
|
55
|
+
- spec/filters/collect_spec.rb
|
56
|
+
- spec/spec_helper.rb
|
57
|
+
homepage: http://mikeski.net
|
58
|
+
licenses:
|
59
|
+
- Apache License (2.0)
|
60
|
+
metadata:
|
61
|
+
logstash_plugin: 'true'
|
62
|
+
logstash_group: filter
|
63
|
+
post_install_message:
|
64
|
+
rdoc_options: []
|
65
|
+
require_paths:
|
66
|
+
- lib
|
67
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: '0'
|
72
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
requirements: []
|
78
|
+
rubyforge_project:
|
79
|
+
rubygems_version: 2.6.8
|
80
|
+
signing_key:
|
81
|
+
specification_version: 4
|
82
|
+
summary: Collect list of properties from list of objects
|
83
|
+
test_files:
|
84
|
+
- spec/filters/collect_spec.rb
|
85
|
+
- spec/spec_helper.rb
|