logstash-filter-urldecode 3.0.2 → 3.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -3
- data/CONTRIBUTORS +1 -0
- data/lib/logstash/filters/urldecode.rb +32 -8
- data/logstash-filter-urldecode.gemspec +1 -1
- data/spec/filters/urldecode_spec.rb +61 -5
- metadata +13 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 65a6db3f0aa1e22c3bbea648cbcbdeb8f3d4aa0e
|
4
|
+
data.tar.gz: 91323d88b743f22fbf15d0c29143b1a01b451752
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 05c0ef24d921d36df65d7955aefac8eae78f68c5f8214e4fe0c3c5de6112513d21bb3aec2eaf1f93f1839c644c8b96bd2c08ab6636e91505b956c7182bd67611
|
7
|
+
data.tar.gz: 25e8b811a4f03967ac4c488eef8d64f6b5b5ef24cb74d272ff9685de7686cea40473a645adea126a612c26a425296d4508de964f9f35363937da7b38267c7c43
|
data/CHANGELOG.md
CHANGED
@@ -1,16 +1,23 @@
|
|
1
|
+
## 3.0.3
|
2
|
+
- Fix 6 "incompatible encodings: UTF-8 and ASCII-8BIT"
|
3
|
+
|
1
4
|
## 3.0.2
|
2
5
|
- Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
|
3
6
|
|
4
7
|
## 3.0.1
|
5
8
|
- Republish all the gems under jruby.
|
9
|
+
|
6
10
|
## 3.0.0
|
7
11
|
- Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
|
8
|
-
|
12
|
+
|
13
|
+
## 2.0.4
|
9
14
|
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
10
|
-
|
15
|
+
|
16
|
+
## 2.0.3
|
11
17
|
- New dependency requirements for logstash-core for the 5.0 release
|
18
|
+
|
12
19
|
## 2.0.0
|
13
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
20
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
14
21
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
15
22
|
- Dependency on logstash-core update to 2.0
|
16
23
|
|
data/CONTRIBUTORS
CHANGED
@@ -7,6 +7,7 @@ Contributors:
|
|
7
7
|
* Jordan Sissel (jordansissel)
|
8
8
|
* Pier-Hugues Pellerin (ph)
|
9
9
|
* Richard Pijnenburg (electrical)
|
10
|
+
* Guy Boertje (guyboertje)
|
10
11
|
|
11
12
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
12
13
|
Logstash, and you aren't on the list above and want to be, please let us know
|
@@ -14,6 +14,9 @@ class LogStash::Filters::Urldecode < LogStash::Filters::Base
|
|
14
14
|
# Urldecode all fields
|
15
15
|
config :all_fields, :validate => :boolean, :default => false
|
16
16
|
|
17
|
+
# Append values to the `tags` field when an exception is thrown
|
18
|
+
config :tag_on_failure, :validate => :array, :default => ["_urldecodefailure"]
|
19
|
+
|
17
20
|
# Thel character encoding used in this filter. Examples include `UTF-8`
|
18
21
|
# and `cp1252`
|
19
22
|
#
|
@@ -21,6 +24,10 @@ class LogStash::Filters::Urldecode < LogStash::Filters::Base
|
|
21
24
|
# or in another character set other than `UTF-8`.
|
22
25
|
config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
|
23
26
|
|
27
|
+
UnescapeRe = /((?:%[0-9a-fA-F]{2})+)/
|
28
|
+
UnescapePercent = '%'.freeze
|
29
|
+
UnescapePack = 'H*'.freeze
|
30
|
+
|
24
31
|
public
|
25
32
|
def register
|
26
33
|
@converter = LogStash::Util::Charset.new(@charset)
|
@@ -29,23 +36,40 @@ class LogStash::Filters::Urldecode < LogStash::Filters::Base
|
|
29
36
|
|
30
37
|
public
|
31
38
|
def filter(event)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
39
|
+
begin
|
40
|
+
# If all_fields is true then try to decode them all
|
41
|
+
if @all_fields
|
42
|
+
event.to_hash.each do |name, value|
|
43
|
+
event.set(name, urldecode(value))
|
44
|
+
end
|
45
|
+
# else decode the specified field
|
46
|
+
else
|
47
|
+
event.set(field, urldecode(event.get(@field)))
|
48
|
+
end
|
49
|
+
rescue => e
|
50
|
+
@tag_on_failure.each{|tag| event.tag(tag)}
|
38
51
|
end
|
52
|
+
|
39
53
|
filter_matched(event)
|
40
54
|
end # def filter
|
41
55
|
|
56
|
+
private
|
57
|
+
|
58
|
+
def unescape(value)
|
59
|
+
# There is a bug in the Ruby stdlib URI.unescape
|
60
|
+
# this: URI.unescape("europ%C3%A9enneeuropéenne") throws the following error
|
61
|
+
# Encoding::CompatibilityError: incompatible encodings: ASCII-8BIT and UTF-8
|
62
|
+
# because Array#pack returns ASCII-8BIT encoded strings
|
63
|
+
# this a hybrid of URI.unescape and CGI.unescape
|
64
|
+
value.gsub(UnescapeRe){|s|[s.delete(UnescapePercent)].pack(UnescapePack).force_encoding(value.encoding) }
|
65
|
+
end
|
66
|
+
|
42
67
|
# Attempt to handle string, array, and hash values for fields.
|
43
68
|
# For all other datatypes, just return, URI.unescape doesn't support them.
|
44
|
-
private
|
45
69
|
def urldecode(value)
|
46
70
|
case value
|
47
71
|
when String
|
48
|
-
escaped =
|
72
|
+
escaped = unescape(value)
|
49
73
|
return @converter.convert(escaped)
|
50
74
|
when Array
|
51
75
|
ret_values = []
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-urldecode'
|
4
|
-
s.version = '3.0.
|
4
|
+
s.version = '3.0.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The urldecode filter is for decoding fields that are urlencoded."
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -15,7 +15,8 @@ describe LogStash::Filters::Urldecode do
|
|
15
15
|
CONFIG
|
16
16
|
|
17
17
|
sample("message" => "http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode") do
|
18
|
-
|
18
|
+
expect(subject.get("message")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
|
19
|
+
expect(subject.get("tags")).to be_nil
|
19
20
|
end
|
20
21
|
end
|
21
22
|
|
@@ -28,7 +29,8 @@ describe LogStash::Filters::Urldecode do
|
|
28
29
|
CONFIG
|
29
30
|
|
30
31
|
sample("message" => "http://logstash.net/docs/1.3.2/filters/urldecode") do
|
31
|
-
|
32
|
+
expect(subject.get("message")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
|
33
|
+
expect(subject.get("tags")).to be_nil
|
32
34
|
end
|
33
35
|
end
|
34
36
|
|
@@ -44,8 +46,9 @@ describe LogStash::Filters::Urldecode do
|
|
44
46
|
CONFIG
|
45
47
|
|
46
48
|
sample("message" => "http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode", "nonencoded" => "http://logstash.net/docs/1.3.2/filters/urldecode") do
|
47
|
-
|
48
|
-
|
49
|
+
expect(subject.get("message")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
|
50
|
+
expect(subject.get("nonencoded")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
|
51
|
+
expect(subject.get("tags")).to be_nil
|
49
52
|
end
|
50
53
|
end
|
51
54
|
|
@@ -56,7 +59,60 @@ describe LogStash::Filters::Urldecode do
|
|
56
59
|
}
|
57
60
|
CONFIG
|
58
61
|
sample("message" => "/a/sa/search?rgu=0;+%C3%BB%D3%D0%D5%D2%B5%BD=;+%B7%A2%CB%CD=") do
|
59
|
-
|
62
|
+
expect(subject.get("message")).to eq "/a/sa/search?rgu=0;+û\\xD3\\xD0\\xD5ҵ\\xBD=;+\\xB7\\xA2\\xCB\\xCD="
|
63
|
+
expect(subject.get("tags")).to be_nil
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe "urldecode should handle non RFC 3986 compliant strings with encoding in params portion" do
|
68
|
+
config <<-CONFIG
|
69
|
+
filter {
|
70
|
+
urldecode {
|
71
|
+
field => "url"
|
72
|
+
}
|
73
|
+
}
|
74
|
+
CONFIG
|
75
|
+
sample("url" => "/fr/search-results?queryText=Organigramme%20de%20la%20Commission%20europ%C3%A9enne%201998&additionalTextParam=organigramme%20de%20la%20commission%20européenne%201998") do
|
76
|
+
expect(subject.get("url")).to eq "/fr/search-results?queryText=Organigramme de la Commission européenne 1998&additionalTextParam=organigramme de la commission européenne 1998"
|
77
|
+
expect(subject.get("tags")).to be_nil
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
describe "urldecode should handle non RFC 3986 compliant strings with encoding in url portion" do
|
82
|
+
config <<-CONFIG
|
83
|
+
filter {
|
84
|
+
urldecode {
|
85
|
+
field => "url"
|
86
|
+
}
|
87
|
+
}
|
88
|
+
CONFIG
|
89
|
+
sample("url" => "http%3A%2F%2Fl%C3%B8gstash.net%2Fd%C3%B8cs%2F1.3.2%2Ffilters%2Furldecøde?name=frødø%20båggins") do
|
90
|
+
expect(subject.get("url")).to eq "http://løgstash.net/døcs/1.3.2/filters/urldecøde?name=frødø båggins"
|
91
|
+
expect(subject.get("tags")).to be_nil
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
describe "urldecode should handle hashes" do
|
96
|
+
config <<-CONFIG
|
97
|
+
filter {
|
98
|
+
urldecode {}
|
99
|
+
}
|
100
|
+
CONFIG
|
101
|
+
sample("message" => {"url" => "http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode"}) do
|
102
|
+
expect(subject.get("[message][url]")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
|
103
|
+
expect(subject.get("tags")).to be_nil
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
describe "urldecode should handle arrays" do
|
108
|
+
config <<-CONFIG
|
109
|
+
filter {
|
110
|
+
urldecode {}
|
111
|
+
}
|
112
|
+
CONFIG
|
113
|
+
sample("message" => ["http%3A%2F%2Flogstash.net%2Fdocs%2F1.3.2%2Ffilters%2Furldecode"]) do
|
114
|
+
expect(subject.get("[message][0]")).to eq "http://logstash.net/docs/1.3.2/filters/urldecode"
|
115
|
+
expect(subject.get("tags")).to be_nil
|
60
116
|
end
|
61
117
|
end
|
62
118
|
end
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-urldecode
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-04-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core-plugin-api
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - ">="
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - "<="
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: '2.99'
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - ">="
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,20 +28,22 @@ dependencies:
|
|
30
28
|
- - "<="
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: '2.99'
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
|
34
|
+
name: logstash-devutils
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
35
36
|
requirements:
|
36
37
|
- - ">="
|
37
38
|
- !ruby/object:Gem::Version
|
38
39
|
version: '0'
|
39
|
-
|
40
|
-
prerelease: false
|
41
|
-
type: :development
|
42
|
-
version_requirements: !ruby/object:Gem::Requirement
|
40
|
+
requirement: !ruby/object:Gem::Requirement
|
43
41
|
requirements:
|
44
42
|
- - ">="
|
45
43
|
- !ruby/object:Gem::Version
|
46
44
|
version: '0'
|
45
|
+
prerelease: false
|
46
|
+
type: :development
|
47
47
|
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
48
48
|
email: info@elastic.co
|
49
49
|
executables: []
|
@@ -81,7 +81,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
81
81
|
version: '0'
|
82
82
|
requirements: []
|
83
83
|
rubyforge_project:
|
84
|
-
rubygems_version: 2.
|
84
|
+
rubygems_version: 2.4.8
|
85
85
|
signing_key:
|
86
86
|
specification_version: 4
|
87
87
|
summary: The urldecode filter is for decoding fields that are urlencoded.
|