logstash-filter-fingerprint 3.1.2 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/CONTRIBUTORS +2 -0
- data/LICENSE +1 -1
- data/docs/index.asciidoc +10 -12
- data/lib/logstash/filters/fingerprint.rb +29 -28
- data/logstash-filter-fingerprint.gemspec +2 -2
- data/spec/filters/fingerprint_spec.rb +102 -11
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3e5af8a524f7184afb61b5918806aedb56bb8c3e78de31b4eda34bec96166f6c
|
4
|
+
data.tar.gz: beddca5042e663506a3d5310bd82f0968ebf1c6d95fda038711e57339231e674
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4e4058c40531a6e5f33475c7d847b52c64399e884e3dd7feb4585233fe1c91932bc9f0caea3b047b853351e97c83cb8b9edab0af59ff2e94b92cdc64c8ff3e99
|
7
|
+
data.tar.gz: ef3f51852929411c81cd655455611d0dbdd409618ba90df676c2ff2dc65f47803f315046734a4803f858e3bfc6130c0b246c024ed152d8d6c2b2bb205419ce42
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTORS
CHANGED
@@ -8,6 +8,8 @@ Contributors:
|
|
8
8
|
* Richard Pijnenburg (electrical)
|
9
9
|
* Suyog Rao (suyograo)
|
10
10
|
* Tray (torrancew)
|
11
|
+
* praseodym
|
12
|
+
|
11
13
|
|
12
14
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
13
15
|
Logstash, and you aren't on the list above and want to be, please let us know
|
data/LICENSE
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -27,13 +27,10 @@ This can e.g. be used to create consistent document ids when inserting
|
|
27
27
|
events into Elasticsearch, allowing events in Logstash to cause existing
|
28
28
|
documents to be updated rather than new documents to be created.
|
29
29
|
|
30
|
-
NOTE: When using any method other than 'UUID', 'PUNCTUATION' or 'MURMUR3'
|
31
|
-
you must set the key, otherwise the plugin will raise an exception
|
32
|
-
|
33
30
|
NOTE: When the `target` option is set to `UUID` the result won't be
|
34
31
|
a consistent hash but a random
|
35
32
|
https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID].
|
36
|
-
To generate UUIDs, prefer the
|
33
|
+
To generate UUIDs, prefer the {logstash-ref}/plugins-filters-uuid.html[uuid filter].
|
37
34
|
|
38
35
|
[id="plugins-{type}s-{plugin}-options"]
|
39
36
|
==== Fingerprint Filter Configuration Options
|
@@ -80,7 +77,7 @@ fields are given, the target field will be an array with fingerprints
|
|
80
77
|
of the source fields given.
|
81
78
|
|
82
79
|
[id="plugins-{type}s-{plugin}-concatenate_all_fields"]
|
83
|
-
===== `
|
80
|
+
===== `concatenate_all_fields`
|
84
81
|
|
85
82
|
* Value type is <<boolean,boolean>>
|
86
83
|
* Default value is `false`
|
@@ -99,8 +96,7 @@ source fields given.
|
|
99
96
|
* There is no default value for this setting.
|
100
97
|
|
101
98
|
When used with the `IPV4_NETWORK` method fill in the subnet prefix length.
|
102
|
-
|
103
|
-
With other methods fill in the HMAC key.
|
99
|
+
With other methods, optionally fill in the HMAC key.
|
104
100
|
|
105
101
|
[id="plugins-{type}s-{plugin}-method"]
|
106
102
|
===== `method`
|
@@ -111,10 +107,12 @@ With other methods fill in the HMAC key.
|
|
111
107
|
|
112
108
|
The fingerprint method to use.
|
113
109
|
|
114
|
-
If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5`
|
115
|
-
cryptographic
|
116
|
-
|
117
|
-
|
110
|
+
If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
|
111
|
+
the cryptographic hash function with the same name will be used to generate
|
112
|
+
the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
|
113
|
+
be used.
|
114
|
+
|
115
|
+
If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
|
118
116
|
|
119
117
|
If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
|
120
118
|
the hash value will be the masked-out address using the number of bits
|
@@ -150,4 +148,4 @@ Any current contents of that field will be overwritten.
|
|
150
148
|
|
151
149
|
|
152
150
|
[id="plugins-{type}s-{plugin}-common-options"]
|
153
|
-
include::{include_path}/{type}.asciidoc[]
|
151
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -34,8 +34,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
|
34
34
|
config :target, :validate => :string, :default => 'fingerprint'
|
35
35
|
|
36
36
|
# When used with the `IPV4_NETWORK` method fill in the subnet prefix length.
|
37
|
-
#
|
38
|
-
# With other methods fill in the HMAC key.
|
37
|
+
# With other methods, optionally fill in the HMAC key.
|
39
38
|
config :key, :validate => :string
|
40
39
|
|
41
40
|
# When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
|
@@ -44,10 +43,12 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
|
44
43
|
|
45
44
|
# The fingerprint method to use.
|
46
45
|
#
|
47
|
-
# If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5`
|
48
|
-
# cryptographic
|
49
|
-
#
|
50
|
-
#
|
46
|
+
# If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
|
47
|
+
# the cryptographic hash function with the same name will be used to generate
|
48
|
+
# the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
|
49
|
+
# be used.
|
50
|
+
#
|
51
|
+
# If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
|
51
52
|
#
|
52
53
|
# If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
|
53
54
|
# the hash value will be the masked-out address using the number of bits
|
@@ -79,7 +80,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
|
79
80
|
# convert to symbol for faster comparisons
|
80
81
|
@method = @method.to_sym
|
81
82
|
|
82
|
-
# require any library and set the
|
83
|
+
# require any library and set the fingerprint function
|
83
84
|
case @method
|
84
85
|
when :IPV4_NETWORK
|
85
86
|
if @key.nil?
|
@@ -90,23 +91,15 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
|
90
91
|
:error => "Key value is empty. please fill in a subnet prefix length"
|
91
92
|
)
|
92
93
|
end
|
93
|
-
class << self; alias_method :
|
94
|
+
class << self; alias_method :fingerprint, :fingerprint_ipv4_network; end
|
94
95
|
when :MURMUR3
|
95
|
-
class << self; alias_method :
|
96
|
+
class << self; alias_method :fingerprint, :fingerprint_murmur3; end
|
96
97
|
when :UUID
|
97
98
|
# nothing
|
98
99
|
when :PUNCTUATION
|
99
100
|
# nothing
|
100
101
|
else
|
101
|
-
|
102
|
-
raise LogStash::ConfigurationError, I18n.t(
|
103
|
-
"logstash.runner.configuration.invalid_plugin_register",
|
104
|
-
:plugin => "filter",
|
105
|
-
:type => "fingerprint",
|
106
|
-
:error => "Key value is empty. Please fill in an encryption key"
|
107
|
-
)
|
108
|
-
end
|
109
|
-
class << self; alias_method :anonymize, :anonymize_openssl; end
|
102
|
+
class << self; alias_method :fingerprint, :fingerprint_openssl; end
|
110
103
|
@digest = select_digest(@method)
|
111
104
|
end
|
112
105
|
end
|
@@ -137,14 +130,14 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
|
137
130
|
end
|
138
131
|
to_string << "|"
|
139
132
|
@logger.debug? && @logger.debug("String built", :to_checksum => to_string)
|
140
|
-
event.set(@target,
|
133
|
+
event.set(@target, fingerprint(to_string))
|
141
134
|
else
|
142
135
|
@source.each do |field|
|
143
136
|
next unless event.include?(field)
|
144
137
|
if event.get(field).is_a?(Array)
|
145
|
-
event.set(@target, event.get(field).collect { |v|
|
138
|
+
event.set(@target, event.get(field).collect { |v| fingerprint(v) })
|
146
139
|
else
|
147
|
-
event.set(@target,
|
140
|
+
event.set(@target, fingerprint(event.get(field)))
|
148
141
|
end
|
149
142
|
end
|
150
143
|
end
|
@@ -154,22 +147,30 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
|
154
147
|
|
155
148
|
private
|
156
149
|
|
157
|
-
def
|
150
|
+
def fingerprint_ipv4_network(ip_string)
|
158
151
|
# in JRuby 1.7.11 outputs as US-ASCII
|
159
152
|
IPAddr.new(ip_string).mask(@key.to_i).to_s.force_encoding(Encoding::UTF_8)
|
160
153
|
end
|
161
154
|
|
162
|
-
def
|
155
|
+
def fingerprint_openssl(data)
|
163
156
|
# in JRuby 1.7.11 outputs as ASCII-8BIT
|
164
|
-
if @
|
165
|
-
|
166
|
-
|
157
|
+
if @key.nil?
|
158
|
+
if @base64encode
|
159
|
+
@digest.base64digest(data.to_s).force_encoding(Encoding::UTF_8)
|
160
|
+
else
|
161
|
+
@digest.hexdigest(data.to_s).force_encoding(Encoding::UTF_8)
|
162
|
+
end
|
167
163
|
else
|
168
|
-
|
164
|
+
if @base64encode
|
165
|
+
hash = OpenSSL::HMAC.digest(@digest, @key, data.to_s)
|
166
|
+
Base64.strict_encode64(hash).force_encoding(Encoding::UTF_8)
|
167
|
+
else
|
168
|
+
OpenSSL::HMAC.hexdigest(@digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
|
169
|
+
end
|
169
170
|
end
|
170
171
|
end
|
171
172
|
|
172
|
-
def
|
173
|
+
def fingerprint_murmur3(value)
|
173
174
|
case value
|
174
175
|
when Fixnum
|
175
176
|
MurmurHash3::V32.int_hash(value)
|
@@ -1,8 +1,8 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-fingerprint'
|
4
|
-
s.version = '3.
|
5
|
-
s.licenses = ['Apache
|
4
|
+
s.version = '3.2.0'
|
5
|
+
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Fingerprints fields by replacing values with a consistent hash"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
8
8
|
s.authors = ["Elastic"]
|
@@ -35,7 +35,22 @@ describe LogStash::Filters::Fingerprint do
|
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
38
|
-
describe "fingerprint string with SHA1
|
38
|
+
describe "fingerprint string with SHA1 algorithm" do
|
39
|
+
config <<-CONFIG
|
40
|
+
filter {
|
41
|
+
fingerprint {
|
42
|
+
source => ["clientip"]
|
43
|
+
method => 'SHA1'
|
44
|
+
}
|
45
|
+
}
|
46
|
+
CONFIG
|
47
|
+
|
48
|
+
sample("clientip" => "123.123.123.123") do
|
49
|
+
insist { subject.get("fingerprint") } == "3a5076c520b4b463f43806896ea0b3978d09dcae"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe "fingerprint string with SHA1 HMAC algorithm" do
|
39
54
|
config <<-CONFIG
|
40
55
|
filter {
|
41
56
|
fingerprint {
|
@@ -51,7 +66,7 @@ describe LogStash::Filters::Fingerprint do
|
|
51
66
|
end
|
52
67
|
end
|
53
68
|
|
54
|
-
describe "fingerprint string with SHA1
|
69
|
+
describe "fingerprint string with SHA1 HMAC algorithm on all event fields" do
|
55
70
|
config <<-CONFIG
|
56
71
|
filter {
|
57
72
|
fingerprint {
|
@@ -68,7 +83,23 @@ describe LogStash::Filters::Fingerprint do
|
|
68
83
|
end
|
69
84
|
end
|
70
85
|
|
71
|
-
describe "fingerprint string with SHA1
|
86
|
+
describe "fingerprint string with SHA1 algorithm and base64 encoding" do
|
87
|
+
config <<-CONFIG
|
88
|
+
filter {
|
89
|
+
fingerprint {
|
90
|
+
source => ["clientip"]
|
91
|
+
method => 'SHA1'
|
92
|
+
base64encode => true
|
93
|
+
}
|
94
|
+
}
|
95
|
+
CONFIG
|
96
|
+
|
97
|
+
sample("clientip" => "123.123.123.123") do
|
98
|
+
insist { subject.get("fingerprint") } == "OlB2xSC0tGP0OAaJbqCzl40J3K4="
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
describe "fingerprint string with SHA1 HMAC algorithm and base64 encoding" do
|
72
103
|
config <<-CONFIG
|
73
104
|
filter {
|
74
105
|
fingerprint {
|
@@ -85,7 +116,22 @@ describe LogStash::Filters::Fingerprint do
|
|
85
116
|
end
|
86
117
|
end
|
87
118
|
|
88
|
-
describe "fingerprint string with SHA256
|
119
|
+
describe "fingerprint string with SHA256 algorithm" do
|
120
|
+
config <<-CONFIG
|
121
|
+
filter {
|
122
|
+
fingerprint {
|
123
|
+
source => ["clientip"]
|
124
|
+
method => 'SHA256'
|
125
|
+
}
|
126
|
+
}
|
127
|
+
CONFIG
|
128
|
+
|
129
|
+
sample("clientip" => "123.123.123.123") do
|
130
|
+
insist { subject.get("fingerprint") } == "4dabcab210766e35f03e77120e6986d6e6d4752b2a9ff22980b9253d026080d8"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
describe "fingerprint string with SHA256 HMAC algorithm" do
|
89
135
|
config <<-CONFIG
|
90
136
|
filter {
|
91
137
|
fingerprint {
|
@@ -101,7 +147,7 @@ describe LogStash::Filters::Fingerprint do
|
|
101
147
|
end
|
102
148
|
end
|
103
149
|
|
104
|
-
describe "fingerprint string with SHA256
|
150
|
+
describe "fingerprint string with SHA256 HMAC algorithm and base64 encoding" do
|
105
151
|
config <<-CONFIG
|
106
152
|
filter {
|
107
153
|
fingerprint {
|
@@ -118,7 +164,22 @@ describe LogStash::Filters::Fingerprint do
|
|
118
164
|
end
|
119
165
|
end
|
120
166
|
|
121
|
-
describe "fingerprint string with SHA384
|
167
|
+
describe "fingerprint string with SHA384 algorithm" do
|
168
|
+
config <<-CONFIG
|
169
|
+
filter {
|
170
|
+
fingerprint {
|
171
|
+
source => ["clientip"]
|
172
|
+
method => 'SHA384'
|
173
|
+
}
|
174
|
+
}
|
175
|
+
CONFIG
|
176
|
+
|
177
|
+
sample("clientip" => "123.123.123.123") do
|
178
|
+
insist { subject.get("fingerprint") } == "fd605b0a3af3e04ce0d7a0b0d9c48d67a12dab811f60072e6eae84e35d567793ffb68a1807536f11c90874065c2a4392"
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
describe "fingerprint string with SHA384 HMAC algorithm" do
|
122
183
|
config <<-CONFIG
|
123
184
|
filter {
|
124
185
|
fingerprint {
|
@@ -134,7 +195,7 @@ describe LogStash::Filters::Fingerprint do
|
|
134
195
|
end
|
135
196
|
end
|
136
197
|
|
137
|
-
describe "fingerprint string with SHA384
|
198
|
+
describe "fingerprint string with SHA384 HMAC algorithm and base64 encoding" do
|
138
199
|
config <<-CONFIG
|
139
200
|
filter {
|
140
201
|
fingerprint {
|
@@ -151,7 +212,22 @@ describe LogStash::Filters::Fingerprint do
|
|
151
212
|
end
|
152
213
|
end
|
153
214
|
|
154
|
-
describe "fingerprint string with SHA512
|
215
|
+
describe "fingerprint string with SHA512 algorithm" do
|
216
|
+
config <<-CONFIG
|
217
|
+
filter {
|
218
|
+
fingerprint {
|
219
|
+
source => ["clientip"]
|
220
|
+
method => 'SHA512'
|
221
|
+
}
|
222
|
+
}
|
223
|
+
CONFIG
|
224
|
+
|
225
|
+
sample("clientip" => "123.123.123.123") do
|
226
|
+
insist { subject.get("fingerprint") } == "5468e2dc64ea92b617782aae884b35af60041ac9e168a283615b6a462c54c13d42fa9542cce9b7d76a8124ac6616818905e3e5dd35d6e519f77c3b517558639a"
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
describe "fingerprint string with SHA512 HMAC algorithm" do
|
155
231
|
config <<-CONFIG
|
156
232
|
filter {
|
157
233
|
fingerprint {
|
@@ -167,7 +243,7 @@ describe LogStash::Filters::Fingerprint do
|
|
167
243
|
end
|
168
244
|
end
|
169
245
|
|
170
|
-
describe "fingerprint string with SHA512
|
246
|
+
describe "fingerprint string with SHA512 HMAC algorithm and base64 encoding" do
|
171
247
|
config <<-CONFIG
|
172
248
|
filter {
|
173
249
|
fingerprint {
|
@@ -184,7 +260,22 @@ describe LogStash::Filters::Fingerprint do
|
|
184
260
|
end
|
185
261
|
end
|
186
262
|
|
187
|
-
describe "fingerprint string with MD5
|
263
|
+
describe "fingerprint string with MD5 algorithm" do
|
264
|
+
config <<-CONFIG
|
265
|
+
filter {
|
266
|
+
fingerprint {
|
267
|
+
source => ["clientip"]
|
268
|
+
method => 'MD5'
|
269
|
+
}
|
270
|
+
}
|
271
|
+
CONFIG
|
272
|
+
|
273
|
+
sample("clientip" => "123.123.123.123") do
|
274
|
+
insist { subject.get("fingerprint") } == "ccdd8d3d940a01b2fb3258c059924c0d"
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
describe "fingerprint string with MD5 HMAC algorithm" do
|
188
279
|
config <<-CONFIG
|
189
280
|
filter {
|
190
281
|
fingerprint {
|
@@ -200,7 +291,7 @@ describe LogStash::Filters::Fingerprint do
|
|
200
291
|
end
|
201
292
|
end
|
202
293
|
|
203
|
-
describe "fingerprint string with MD5
|
294
|
+
describe "fingerprint string with MD5 HMAC algorithm and base64 encoding" do
|
204
295
|
config <<-CONFIG
|
205
296
|
filter {
|
206
297
|
fingerprint {
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-fingerprint
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-06-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -78,7 +78,7 @@ files:
|
|
78
78
|
- spec/filters/fingerprint_spec.rb
|
79
79
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
80
80
|
licenses:
|
81
|
-
- Apache
|
81
|
+
- Apache-2.0
|
82
82
|
metadata:
|
83
83
|
logstash_plugin: 'true'
|
84
84
|
logstash_group: filter
|
@@ -98,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
98
98
|
version: '0'
|
99
99
|
requirements: []
|
100
100
|
rubyforge_project:
|
101
|
-
rubygems_version: 2.6.
|
101
|
+
rubygems_version: 2.6.13
|
102
102
|
signing_key:
|
103
103
|
specification_version: 4
|
104
104
|
summary: Fingerprints fields by replacing values with a consistent hash
|