logstash-filter-fingerprint 2.0.2 → 2.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/README.md +3 -0
- data/lib/logstash/filters/fingerprint.rb +86 -79
- data/logstash-filter-fingerprint.gemspec +1 -1
- metadata +15 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3ce813384001a6819fb6fab6008531c3e0a0973a
|
4
|
+
data.tar.gz: f684a7c3bd2bb062fad4a437169acdd7f89432f3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b902b68506a934ac849ce8be94a6a4fb9913efc30d95c8d656c4e9008ae78ac25ba35849c5e1d5a2c2035ef18e7f584bae903868989debbbafcdf2f0ab8e6a9b
|
7
|
+
data.tar.gz: 28f652c9adde684c87e6adcd0864160f8f2730605eab0be29e979d7f15d9a73f3bcf2d2f555f1913f6ccfa39e2e7df44cc7ed84da689235d00b7d7bb42a48ca1
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
+
## 2.0.3
|
2
|
+
- Eager loading of libraries, optimizations and cleanups https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/10
|
3
|
+
|
1
4
|
## 2.0.0
|
2
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
5
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
6
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
7
|
- Dependency on logstash-core update to 2.0
|
5
8
|
|
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
+
[![Build
|
4
|
+
Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-fingerprint-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-fingerprint-unit/)
|
5
|
+
|
3
6
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
7
|
|
5
8
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
@@ -2,6 +2,10 @@
|
|
2
2
|
require "logstash/filters/base"
|
3
3
|
require "logstash/namespace"
|
4
4
|
require "base64"
|
5
|
+
require "openssl"
|
6
|
+
require "ipaddr"
|
7
|
+
require "murmurhash3"
|
8
|
+
require "securerandom"
|
5
9
|
|
6
10
|
# Fingerprint fields using by replacing values with a consistent hash.
|
7
11
|
class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
@@ -30,113 +34,116 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
|
|
30
34
|
config :concatenate_sources, :validate => :boolean, :default => false
|
31
35
|
|
32
36
|
def register
|
37
|
+
# convert to symbol for faster comparisons
|
38
|
+
@method = @method.to_sym
|
39
|
+
|
33
40
|
# require any library and set the anonymize function
|
34
41
|
case @method
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
:
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
42
|
+
when :IPV4_NETWORK
|
43
|
+
if @key.nil?
|
44
|
+
raise LogStash::ConfigurationError, I18n.t(
|
45
|
+
"logstash.agent.configuration.invalid_plugin_register",
|
46
|
+
:plugin => "filter",
|
47
|
+
:type => "fingerprint",
|
48
|
+
:error => "Key value is empty. please fill in a subnet prefix length"
|
49
|
+
)
|
50
|
+
end
|
51
|
+
class << self; alias_method :anonymize, :anonymize_ipv4_network; end
|
52
|
+
when :MURMUR3
|
53
|
+
class << self; alias_method :anonymize, :anonymize_murmur3; end
|
54
|
+
when :UUID
|
55
|
+
# nothing
|
56
|
+
when :PUNCTUATION
|
57
|
+
# nothing
|
58
|
+
else
|
59
|
+
if @key.nil?
|
60
|
+
raise LogStash::ConfigurationError, I18n.t(
|
61
|
+
"logstash.agent.configuration.invalid_plugin_register",
|
62
|
+
:plugin => "filter",
|
63
|
+
:type => "fingerprint",
|
64
|
+
:error => "Key value is empty. Please fill in an encryption key"
|
65
|
+
)
|
66
|
+
end
|
67
|
+
class << self; alias_method :anonymize, :anonymize_openssl; end
|
68
|
+
@digest = select_digest(@method)
|
58
69
|
end
|
59
|
-
end
|
70
|
+
end
|
60
71
|
|
61
|
-
public
|
62
72
|
def filter(event)
|
63
|
-
|
64
73
|
case @method
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
+
when :UUID
|
75
|
+
event[@target] = SecureRandom.uuid
|
76
|
+
when :PUNCTUATION
|
77
|
+
@source.sort.each do |field|
|
78
|
+
next unless event.include?(field)
|
79
|
+
# In order to keep some backwards compatibility we should use the unicode version
|
80
|
+
# of the regexp because the POSIX one ([[:punct:]]) left some unwanted characters unfiltered (Symbols).
|
81
|
+
# gsub(/[^[:punct:]]/,'') should be equivalent to gsub(/[^[\p{P}\p{S}]]/,''), but not 100% in JRuby.
|
82
|
+
event[@target] = event[field].gsub(/[^[\p{P}\p{S}]]/,'')
|
83
|
+
end
|
84
|
+
else
|
85
|
+
if @concatenate_sources
|
86
|
+
to_string = ""
|
87
|
+
@source.sort.each do |k|
|
88
|
+
to_string << "|#{k}|#{event[k]}"
|
74
89
|
end
|
90
|
+
to_string << "|"
|
91
|
+
@logger.debug? && @logger.debug("String built", :to_checksum => to_string)
|
92
|
+
event[@target] = anonymize(to_string)
|
75
93
|
else
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
@
|
80
|
-
|
94
|
+
@source.each do |field|
|
95
|
+
next unless event.include?(field)
|
96
|
+
if event[field].is_a?(Array)
|
97
|
+
event[@target] = event[field].collect { |v| anonymize(v) }
|
98
|
+
else
|
99
|
+
event[@target] = anonymize(event[field])
|
81
100
|
end
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
@source.each do |field|
|
87
|
-
next unless event.include?(field)
|
88
|
-
if event[field].is_a?(Array)
|
89
|
-
event[@target] = event[field].collect { |v| anonymize(v) }
|
90
|
-
else
|
91
|
-
event[@target] = anonymize(event[field])
|
92
|
-
end
|
93
|
-
end # @source.each
|
94
|
-
end # concatenate_sources
|
95
|
-
|
96
|
-
end # casse @method
|
97
|
-
end # def filter
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
98
105
|
|
99
106
|
private
|
107
|
+
|
100
108
|
def anonymize_ipv4_network(ip_string)
|
101
109
|
# in JRuby 1.7.11 outputs as US-ASCII
|
102
110
|
IPAddr.new(ip_string).mask(@key.to_i).to_s.force_encoding(Encoding::UTF_8)
|
103
111
|
end
|
104
112
|
|
105
113
|
def anonymize_openssl(data)
|
106
|
-
digest = encryption_algorithm()
|
107
114
|
# in JRuby 1.7.11 outputs as ASCII-8BIT
|
108
115
|
if @base64encode
|
109
|
-
hash = OpenSSL::HMAC.digest(digest, @key, data.to_s)
|
116
|
+
hash = OpenSSL::HMAC.digest(@digest, @key, data.to_s)
|
110
117
|
Base64.strict_encode64(hash).force_encoding(Encoding::UTF_8)
|
111
118
|
else
|
112
|
-
OpenSSL::HMAC.hexdigest(digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
|
119
|
+
OpenSSL::HMAC.hexdigest(@digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
|
113
120
|
end
|
114
121
|
end
|
115
122
|
|
116
123
|
def anonymize_murmur3(value)
|
117
124
|
case value
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
125
|
+
when Fixnum
|
126
|
+
MurmurHash3::V32.int_hash(value)
|
127
|
+
else
|
128
|
+
MurmurHash3::V32.str_hash(value.to_s)
|
122
129
|
end
|
123
130
|
end
|
124
131
|
|
125
|
-
def
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
132
|
+
def select_digest(method)
|
133
|
+
case method
|
134
|
+
when :SHA1
|
135
|
+
OpenSSL::Digest::SHA1.new
|
136
|
+
when :SHA256
|
137
|
+
OpenSSL::Digest::SHA256.new
|
138
|
+
when :SHA384
|
139
|
+
OpenSSL::Digest::SHA384.new
|
140
|
+
when :SHA512
|
141
|
+
OpenSSL::Digest::SHA512.new
|
142
|
+
when :MD5
|
143
|
+
OpenSSL::Digest::MD5.new
|
144
|
+
else
|
145
|
+
# we really should never get here
|
146
|
+
raise(LogStash::ConfigurationError, "Unknown digest for method=#{method.to_s}")
|
139
147
|
end
|
140
148
|
end
|
141
|
-
|
142
|
-
end # class LogStash::Filters::Anonymize
|
149
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-fingerprint'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Fingerprint fields using by replacing values with a consistent hash."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-fingerprint
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,34 +28,36 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: murmurhash3
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: murmurhash3
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: logstash-devutils
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
51
|
- - '>='
|
45
52
|
- !ruby/object:Gem::Version
|
46
53
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - '>='
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: '0'
|
53
|
-
name: logstash-devutils
|
54
59
|
prerelease: false
|
55
60
|
type: :development
|
56
|
-
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
requirements:
|
58
|
-
- - '>='
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
version: '0'
|
61
61
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
62
62
|
email: info@elastic.co
|
63
63
|
executables: []
|