fluent-plugin-uri-parser 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -12
- data/fluent-plugin-uri-parser.gemspec +3 -3
- data/lib/fluent/plugin/filter_query_string_parser.rb +43 -42
- data/lib/fluent/plugin/filter_uri_parser.rb +53 -51
- metadata +12 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 529c3e22f0bb6e97190a8db3c9bccd8005db2eb0
|
|
4
|
+
data.tar.gz: 7a61cff618e86c55178494fd51a528ea7daedf80
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c4e6a42fc6d43a03d32c45e7017a16597e6354e1a3b05508a61f58b57ea5da060236f314abe447b454b010f17f5cba639a6af377e7d5123d94f599c1adefacb1
|
|
7
|
+
data.tar.gz: bfc2f8c5f4184ed49fc746e564ae5aa97428b071c7c0929001a51c3835896b6598a0fb276167db76afe2a1668fbd2e040bf47d737fa08d14efa14605afe78ac6
|
data/README.md
CHANGED
|
@@ -4,21 +4,19 @@
|
|
|
4
4
|
|
|
5
5
|
This is a Fluentd plugin to parse uri and query string in log messages.
|
|
6
6
|
|
|
7
|
-
##
|
|
8
|
-
|
|
9
|
-
Add this line to your application's Gemfile:
|
|
10
|
-
|
|
11
|
-
```ruby
|
|
12
|
-
gem 'fluent-plugin-uri-parser'
|
|
13
|
-
```
|
|
7
|
+
## Requirements
|
|
14
8
|
|
|
15
|
-
|
|
9
|
+
| fluent-plugin-uri-parser | fluentd | ruby |
|
|
10
|
+
|---------------------------|------------|--------|
|
|
11
|
+
| >= 0.3.0 | >= v0.14.0 | >= 2.1 |
|
|
12
|
+
| < 0.2.0 | >= v0.12.0 | >= 1.9 |
|
|
16
13
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
Or install it yourself as:
|
|
14
|
+
## Installation
|
|
20
15
|
|
|
21
|
-
|
|
16
|
+
``` shell
|
|
17
|
+
$ gem install fluent-plugin-uri-parser -v "~> 0.2" # for fluentd v0.12 or later
|
|
18
|
+
$ gem install fluent-plugin-uri-parser # for fluentd v0.14 or later
|
|
19
|
+
```
|
|
22
20
|
|
|
23
21
|
## Component
|
|
24
22
|
|
|
@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-uri-parser"
|
|
7
|
-
spec.version = "0.
|
|
7
|
+
spec.version = "0.3.0"
|
|
8
8
|
spec.authors = ["Daichi HIRATA"]
|
|
9
|
-
spec.email = ["
|
|
9
|
+
spec.email = ["daichirata@gmail.com"]
|
|
10
10
|
spec.license = "Apache-2.0"
|
|
11
11
|
|
|
12
12
|
spec.summary = "This is a Fluentd plugin to parse uri and query string in log messages."
|
|
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
|
17
17
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
18
18
|
spec.require_paths = ["lib"]
|
|
19
19
|
|
|
20
|
-
spec.add_runtime_dependency "fluentd", "
|
|
20
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
|
|
21
21
|
spec.add_runtime_dependency "addressable"
|
|
22
22
|
|
|
23
23
|
spec.add_development_dependency "test-unit"
|
|
@@ -1,53 +1,54 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
new_es = Fluent::MultiEventStream.new
|
|
18
|
-
|
|
19
|
-
es.each do |time, record|
|
|
20
|
-
raw_value = record[@key_name]
|
|
21
|
-
if raw_value.nil?
|
|
22
|
-
new_es.add(time, record) unless @ignore_key_not_exist
|
|
23
|
-
next
|
|
1
|
+
module Fluent
|
|
2
|
+
module Plugin
|
|
3
|
+
class QueryStringParserFilter < Filter
|
|
4
|
+
Fluent::Plugin.register_filter("query_string_parser", self)
|
|
5
|
+
|
|
6
|
+
config_param :key_name, :string
|
|
7
|
+
config_param :hash_value_field, :string, default: nil
|
|
8
|
+
config_param :inject_key_prefix, :string, default: nil
|
|
9
|
+
config_param :suppress_parse_error_log, :bool, default: false
|
|
10
|
+
config_param :ignore_key_not_exist, :bool, default: false
|
|
11
|
+
config_param :emit_invalid_record_to_error, :bool, default: true
|
|
12
|
+
config_param :multi_value_params, :bool, default: false
|
|
13
|
+
|
|
14
|
+
def initialize
|
|
15
|
+
super
|
|
16
|
+
require "uri"
|
|
24
17
|
end
|
|
25
18
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
unless params.empty?
|
|
19
|
+
def filter(tag, time, record)
|
|
20
|
+
raw_value = record[@key_name]
|
|
30
21
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
else
|
|
35
|
-
values = Hash[params]
|
|
22
|
+
if raw_value.nil?
|
|
23
|
+
if @emit_invalid_record_to_error
|
|
24
|
+
router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
|
|
36
25
|
end
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
26
|
+
return @ignore_key_not_exist ? nil : record
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
begin
|
|
30
|
+
params = URI.decode_www_form(raw_value)
|
|
31
|
+
|
|
32
|
+
unless params.empty?
|
|
33
|
+
if @multi_value_params
|
|
34
|
+
values = Hash.new {|h,k| h[k] = [] }
|
|
35
|
+
params.each{|pair| values[pair[0]].push(pair[1])}
|
|
36
|
+
else
|
|
37
|
+
values = Hash[params]
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
if @inject_key_prefix
|
|
41
|
+
values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
|
|
42
|
+
end
|
|
43
|
+
r = @hash_value_field ? { @hash_value_field => values } : values
|
|
44
|
+
record = record.merge(r)
|
|
40
45
|
end
|
|
41
|
-
|
|
42
|
-
|
|
46
|
+
rescue => e
|
|
47
|
+
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
43
48
|
end
|
|
44
49
|
|
|
45
|
-
|
|
46
|
-
rescue => e
|
|
47
|
-
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
50
|
+
return record
|
|
48
51
|
end
|
|
49
52
|
end
|
|
50
|
-
|
|
51
|
-
new_es
|
|
52
53
|
end
|
|
53
54
|
end
|
|
@@ -1,61 +1,63 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
1
|
+
module Fluent
|
|
2
|
+
module Plugin
|
|
3
|
+
class URIParserFilter < Filter
|
|
4
|
+
Fluent::Plugin.register_filter("uri_parser", self)
|
|
5
|
+
|
|
6
|
+
config_param :key_name, :string
|
|
7
|
+
config_param :hash_value_field, :string, default: nil
|
|
8
|
+
config_param :inject_key_prefix, :string, default: nil
|
|
9
|
+
config_param :suppress_parse_error_log, :bool, default: false
|
|
10
|
+
config_param :ignore_key_not_exist, :bool, default: false
|
|
11
|
+
config_param :ignore_nil, :bool, default: false
|
|
12
|
+
config_param :emit_invalid_record_to_error, :bool, default: true
|
|
13
|
+
|
|
14
|
+
config_param :out_key_scheme, :string, default: nil
|
|
15
|
+
config_param :out_key_host, :string, default: nil
|
|
16
|
+
config_param :out_key_port, :string, default: nil
|
|
17
|
+
config_param :out_key_path, :string, default: nil
|
|
18
|
+
config_param :out_key_query, :string, default: nil
|
|
19
|
+
config_param :out_key_fragment, :string, default: nil
|
|
20
|
+
|
|
21
|
+
def initialize
|
|
22
|
+
super
|
|
23
|
+
require "addressable/uri"
|
|
24
|
+
end
|
|
22
25
|
|
|
23
|
-
|
|
24
|
-
|
|
26
|
+
def filter(tag, time, record)
|
|
27
|
+
raw_value = record[@key_name]
|
|
25
28
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
if raw_value.nil?
|
|
30
|
+
if @emit_invalid_record_to_error
|
|
31
|
+
router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
|
|
32
|
+
end
|
|
33
|
+
return @ignore_key_not_exist ? nil : record
|
|
34
|
+
end
|
|
32
35
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
36
|
+
begin
|
|
37
|
+
uri = Addressable::URI.parse(raw_value)
|
|
38
|
+
|
|
39
|
+
values = {}
|
|
40
|
+
values[@out_key_scheme] = uri.scheme if @out_key_scheme
|
|
41
|
+
values[@out_key_host] = uri.host if @out_key_host
|
|
42
|
+
values[@out_key_port] = uri.inferred_port if @out_key_port
|
|
43
|
+
values[@out_key_path] = uri.path if @out_key_path
|
|
44
|
+
values[@out_key_query] = uri.query if @out_key_query
|
|
45
|
+
values[@out_key_fragment] = uri.fragment if @out_key_fragment
|
|
46
|
+
values.reject! {|_, v| v.nil? } if @ignore_nil
|
|
47
|
+
|
|
48
|
+
unless values.empty?
|
|
49
|
+
if @inject_key_prefix
|
|
50
|
+
values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
|
|
51
|
+
end
|
|
52
|
+
r = @hash_value_field ? { @hash_value_field => values } : values
|
|
53
|
+
record = record.merge(r)
|
|
48
54
|
end
|
|
49
|
-
|
|
50
|
-
|
|
55
|
+
rescue => e
|
|
56
|
+
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
51
57
|
end
|
|
52
58
|
|
|
53
|
-
|
|
54
|
-
rescue => e
|
|
55
|
-
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
59
|
+
return record
|
|
56
60
|
end
|
|
57
61
|
end
|
|
58
|
-
|
|
59
|
-
new_es
|
|
60
62
|
end
|
|
61
63
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-uri-parser
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daichi HIRATA
|
|
@@ -14,16 +14,22 @@ dependencies:
|
|
|
14
14
|
name: fluentd
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - "
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: 0.14.0
|
|
20
|
+
- - "<"
|
|
18
21
|
- !ruby/object:Gem::Version
|
|
19
|
-
version:
|
|
22
|
+
version: '2'
|
|
20
23
|
type: :runtime
|
|
21
24
|
prerelease: false
|
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
26
|
requirements:
|
|
24
|
-
- - "
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: 0.14.0
|
|
30
|
+
- - "<"
|
|
25
31
|
- !ruby/object:Gem::Version
|
|
26
|
-
version:
|
|
32
|
+
version: '2'
|
|
27
33
|
- !ruby/object:Gem::Dependency
|
|
28
34
|
name: addressable
|
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -82,7 +88,7 @@ dependencies:
|
|
|
82
88
|
version: '10.0'
|
|
83
89
|
description: This is a Fluentd plugin to parse uri and query string in log messages.
|
|
84
90
|
email:
|
|
85
|
-
-
|
|
91
|
+
- daichirata@gmail.com
|
|
86
92
|
executables: []
|
|
87
93
|
extensions: []
|
|
88
94
|
extra_rdoc_files: []
|