logstash-filter-prune 3.0.0 → 3.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +10 -1
- data/docs/index.asciidoc +154 -0
- data/logstash-filter-prune.gemspec +2 -2
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0b06d3ca2d9d6fd300c3f9e78bcfcacacf638b16
|
4
|
+
data.tar.gz: 091aefc03fe5f1ac0557959c01130e4ef237d008
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 25a08cb1c3211db98bc2699d9469071a7be3a5634ecae82b313d6bee6a95aad6918cf089aa1c754b5ec8175166f11c231d2fefc56391dee42bb7b5543c065165
|
7
|
+
data.tar.gz: c6330872232aa93ae4961f5727271c1e0eb0d25489ecb2882413a46007080ceff5adf89f58405edf0fe222accffa4ca3270f701dffccb5530d8a592fdac6ac3e
|
data/Gemfile
CHANGED
@@ -1,2 +1,11 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
|
-
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
:plugin: prune
|
2
|
+
:type: filter
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Prune filter plugin
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
The prune filter is for removing fields from events based on
|
24
|
+
whitelists or blacklist of field names or their values (names and
|
25
|
+
values can also be regular expressions).
|
26
|
+
|
27
|
+
This can e.g. be useful if you have a <<plugins-filters-json,json>>
|
28
|
+
or <<plugins-filters-kv,kv>> filter that creates a number of fields
|
29
|
+
with names that you don't necessarily know the names of beforehand,
|
30
|
+
and you only want to keep a subset of them.
|
31
|
+
|
32
|
+
Usage help:
|
33
|
+
To specify a exact field name or value use the regular expression syntax `^some_name_or_value$`.
|
34
|
+
Example usage: Input data `{ "msg":"hello world", "msg_short":"hw" }`
|
35
|
+
[source,ruby]
|
36
|
+
filter {
|
37
|
+
prune {
|
38
|
+
whitelist_names => [ "msg" ]
|
39
|
+
}
|
40
|
+
}
|
41
|
+
Allows both `"msg"` and `"msg_short"` through.
|
42
|
+
|
43
|
+
While:
|
44
|
+
[source,ruby]
|
45
|
+
filter {
|
46
|
+
prune {
|
47
|
+
whitelist_names => ["^msg$"]
|
48
|
+
}
|
49
|
+
}
|
50
|
+
Allows only `"msg"` through.
|
51
|
+
|
52
|
+
Logstash stores an event's `tags` as a field which is subject to pruning. Remember to `whitelist_names => [ "^tags$" ]`
|
53
|
+
to maintain `tags` after pruning or use `blacklist_values => [ "^tag_name$" ]` to eliminate a specific `tag`.
|
54
|
+
|
55
|
+
NOTE: This filter currently only support operations on top-level fields,
|
56
|
+
i.e. whitelisting and blacklisting of subfields based on name or value
|
57
|
+
does not work.
|
58
|
+
|
59
|
+
[id="plugins-{type}s-{plugin}-options"]
|
60
|
+
==== Prune Filter Configuration Options
|
61
|
+
|
62
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
63
|
+
|
64
|
+
[cols="<,<,<",options="header",]
|
65
|
+
|=======================================================================
|
66
|
+
|Setting |Input type|Required
|
67
|
+
| <<plugins-{type}s-{plugin}-blacklist_names>> |<<array,array>>|No
|
68
|
+
| <<plugins-{type}s-{plugin}-blacklist_values>> |<<hash,hash>>|No
|
69
|
+
| <<plugins-{type}s-{plugin}-interpolate>> |<<boolean,boolean>>|No
|
70
|
+
| <<plugins-{type}s-{plugin}-whitelist_names>> |<<array,array>>|No
|
71
|
+
| <<plugins-{type}s-{plugin}-whitelist_values>> |<<hash,hash>>|No
|
72
|
+
|=======================================================================
|
73
|
+
|
74
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
75
|
+
filter plugins.
|
76
|
+
|
77
|
+
|
78
|
+
|
79
|
+
[id="plugins-{type}s-{plugin}-blacklist_names"]
|
80
|
+
===== `blacklist_names`
|
81
|
+
|
82
|
+
* Value type is <<array,array>>
|
83
|
+
* Default value is `["%{[^}]+}"]`
|
84
|
+
|
85
|
+
Exclude fields whose names match specified regexps, by default exclude unresolved `%{field}` strings.
|
86
|
+
[source,ruby]
|
87
|
+
filter {
|
88
|
+
prune {
|
89
|
+
blacklist_names => [ "method", "(referrer|status)", "${some}_field" ]
|
90
|
+
}
|
91
|
+
}
|
92
|
+
|
93
|
+
[id="plugins-{type}s-{plugin}-blacklist_values"]
|
94
|
+
===== `blacklist_values`
|
95
|
+
|
96
|
+
* Value type is <<hash,hash>>
|
97
|
+
* Default value is `{}`
|
98
|
+
|
99
|
+
Exclude specified fields if their values match one of the supplied regular expressions.
|
100
|
+
In case field values are arrays, each array item is matched against the regular expressions and matching array items will be excluded.
|
101
|
+
[source,ruby]
|
102
|
+
filter {
|
103
|
+
prune {
|
104
|
+
blacklist_values => [ "uripath", "/index.php",
|
105
|
+
"method", "(HEAD|OPTIONS)",
|
106
|
+
"status", "^[^2]" ]
|
107
|
+
}
|
108
|
+
}
|
109
|
+
|
110
|
+
[id="plugins-{type}s-{plugin}-interpolate"]
|
111
|
+
===== `interpolate`
|
112
|
+
|
113
|
+
* Value type is <<boolean,boolean>>
|
114
|
+
* Default value is `false`
|
115
|
+
|
116
|
+
Trigger whether configuration fields and values should be interpolated for
|
117
|
+
dynamic values (when resolving `%{some_field}`).
|
118
|
+
Probably adds some performance overhead. Defaults to false.
|
119
|
+
|
120
|
+
[id="plugins-{type}s-{plugin}-whitelist_names"]
|
121
|
+
===== `whitelist_names`
|
122
|
+
|
123
|
+
* Value type is <<array,array>>
|
124
|
+
* Default value is `[]`
|
125
|
+
|
126
|
+
Include only fields only if their names match specified regexps, default to empty list which means include everything.
|
127
|
+
[source,ruby]
|
128
|
+
filter {
|
129
|
+
prune {
|
130
|
+
whitelist_names => [ "method", "(referrer|status)", "${some}_field" ]
|
131
|
+
}
|
132
|
+
}
|
133
|
+
|
134
|
+
[id="plugins-{type}s-{plugin}-whitelist_values"]
|
135
|
+
===== `whitelist_values`
|
136
|
+
|
137
|
+
* Value type is <<hash,hash>>
|
138
|
+
* Default value is `{}`
|
139
|
+
|
140
|
+
Include specified fields only if their values match one of the supplied regular expressions.
|
141
|
+
In case field values are arrays, each array item is matched against the regular expressions and only matching array items will be included.
|
142
|
+
[source,ruby]
|
143
|
+
filter {
|
144
|
+
prune {
|
145
|
+
whitelist_values => [ "uripath", "/index.php",
|
146
|
+
"method", "(GET|POST)",
|
147
|
+
"status", "^[^2]" ]
|
148
|
+
}
|
149
|
+
}
|
150
|
+
|
151
|
+
|
152
|
+
|
153
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
154
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-prune'
|
4
|
-
s.version = '3.0.
|
4
|
+
s.version = '3.0.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The prune filter is for pruning event data from fields based on whitelist/blacklist of field names or their values (names and values can also be regular expressions)"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.require_paths = ["lib"]
|
12
12
|
|
13
13
|
# Files
|
14
|
-
s.files = Dir[
|
14
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
15
15
|
|
16
16
|
# Tests
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-prune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -56,6 +56,7 @@ files:
|
|
56
56
|
- LICENSE
|
57
57
|
- NOTICE.TXT
|
58
58
|
- README.md
|
59
|
+
- docs/index.asciidoc
|
59
60
|
- lib/logstash/filters/prune.rb
|
60
61
|
- logstash-filter-prune.gemspec
|
61
62
|
- spec/filters/prune_spec.rb
|