fluent-plugin-uri-parser 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/test.yml +28 -0
- data/.gitignore +1 -0
- data/README.md +216 -56
- data/fluent-plugin-uri-parser.gemspec +8 -6
- data/lib/fluent/plugin/filter_query_string_parser.rb +55 -45
- data/lib/fluent/plugin/filter_uri_parser.rb +53 -53
- metadata +26 -21
- data/wercker.yml +0 -25
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 159e54e1d62cc98d696b44b59550cb54fd02f9a202afcdf55e5e8df2c7f39996
|
|
4
|
+
data.tar.gz: ed7ee9ce893e2e0cdbbff3fe8ab616b7efcb23db1071a1809a2bb9021cf1d1ad
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0a91925202870044e0099870e59de97670fc80591373b0d13db486827d5e8104188f4828f7ce13fcc4d731ee9b5e887c3320338b28bd3abf84029273eba8bbae
|
|
7
|
+
data.tar.gz: 9968c9681ebccc76584994f3b5479d62ce12a4c7e41f522f428c38697fc2374a453853082ab72f94eb96813bba2388e43f7a88944417d2a29537733f43d17400
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [master]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [master]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
ruby:
|
|
16
|
+
- "3.2"
|
|
17
|
+
- "3.3"
|
|
18
|
+
- "3.4"
|
|
19
|
+
- "4.0"
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v4
|
|
22
|
+
- name: Set up Ruby
|
|
23
|
+
uses: ruby/setup-ruby@v1
|
|
24
|
+
with:
|
|
25
|
+
ruby-version: ${{ matrix.ruby }}
|
|
26
|
+
bundler-cache: true
|
|
27
|
+
- name: Run tests
|
|
28
|
+
run: bundle exec rake test
|
data/.gitignore
CHANGED
data/README.md
CHANGED
|
@@ -1,106 +1,266 @@
|
|
|
1
1
|
# fluent-plugin-uri-parser
|
|
2
2
|
|
|
3
|
-
[](https://badge.fury.io/rb/fluent-plugin-uri-parser)
|
|
3
|
+
[](https://badge.fury.io/rb/fluent-plugin-uri-parser)
|
|
4
|
+
[](https://github.com/daichirata/fluent-plugin-uri-parser/actions/workflows/test.yml)
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
Fluentd filter plugins that **decompose URIs and query strings into structured fields**, so you can search, group, and aggregate on them in your downstream stack.
|
|
7
|
+
|
|
8
|
+
```text
|
|
9
|
+
"https://example.com:8080/search?q=fluentd&lang=ja"
|
|
10
|
+
↓
|
|
11
|
+
{ scheme: "https", host: "example.com", port: 8080,
|
|
12
|
+
path: "/search", query: "q=fluentd&lang=ja", fragment: nil }
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Why?
|
|
16
|
+
|
|
17
|
+
URL strings sitting in a single log field are a black box — you can't filter by host, group by path, or count by query parameter without parsing them first. These filters turn a raw URL into a record your storage can index.
|
|
18
|
+
|
|
19
|
+
| Plugin | Turns this | Into this |
|
|
20
|
+
|---|---|---|
|
|
21
|
+
| `uri_parser` | `https://example.com:8080/p?q=1#x` | `scheme`, `host`, `port`, `path`, `query`, `fragment` |
|
|
22
|
+
| `query_string_parser` | `foo=bar&hoge=fuga` | `{ "foo" => "bar", "hoge" => "fuga" }` |
|
|
23
|
+
|
|
24
|
+
## Requirements
|
|
25
|
+
|
|
26
|
+
| fluent-plugin-uri-parser | fluentd | ruby |
|
|
27
|
+
| ------------------------ | --------- | ------ |
|
|
28
|
+
| >= 0.4.0 | >= v1.0.0 | >= 3.2 |
|
|
6
29
|
|
|
7
30
|
## Installation
|
|
8
31
|
|
|
9
|
-
|
|
32
|
+
```shell
|
|
33
|
+
gem install fluent-plugin-uri-parser
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Or in your `Gemfile`:
|
|
10
37
|
|
|
11
38
|
```ruby
|
|
12
|
-
gem
|
|
39
|
+
gem "fluent-plugin-uri-parser"
|
|
13
40
|
```
|
|
14
41
|
|
|
15
|
-
|
|
42
|
+
---
|
|
16
43
|
|
|
17
|
-
|
|
44
|
+
## `uri_parser`
|
|
18
45
|
|
|
19
|
-
|
|
46
|
+
Decomposes a URI field into its components.
|
|
20
47
|
|
|
21
|
-
|
|
48
|
+
### Minimal example
|
|
22
49
|
|
|
23
|
-
|
|
50
|
+
```aconf
|
|
51
|
+
<filter access.log>
|
|
52
|
+
@type uri_parser
|
|
53
|
+
key_name url
|
|
54
|
+
out_key_scheme scheme
|
|
55
|
+
out_key_host host
|
|
56
|
+
out_key_port port
|
|
57
|
+
out_key_path path
|
|
58
|
+
out_key_query query
|
|
59
|
+
out_key_fragment fragment
|
|
60
|
+
</filter>
|
|
61
|
+
```
|
|
24
62
|
|
|
25
|
-
|
|
63
|
+
```jsonc
|
|
64
|
+
// input
|
|
65
|
+
{ "url": "https://example.com:8080/search?q=fluentd#top" }
|
|
66
|
+
|
|
67
|
+
// output
|
|
68
|
+
{
|
|
69
|
+
"url": "https://example.com:8080/search?q=fluentd#top",
|
|
70
|
+
"scheme": "https",
|
|
71
|
+
"host": "example.com",
|
|
72
|
+
"port": 8080,
|
|
73
|
+
"path": "/search",
|
|
74
|
+
"query": "q=fluentd",
|
|
75
|
+
"fragment": "top"
|
|
76
|
+
}
|
|
77
|
+
```
|
|
26
78
|
|
|
27
|
-
|
|
79
|
+
> The `port` value uses [`Addressable::URI#inferred_port`](https://github.com/sporkmonger/addressable), so well-known schemes (`http` → 80, `https` → 443, ...) get a port even when the URL omits one.
|
|
28
80
|
|
|
29
|
-
###
|
|
81
|
+
### Group output under a single key — `hash_value_field`
|
|
30
82
|
|
|
31
|
-
|
|
83
|
+
```aconf
|
|
84
|
+
<filter access.log>
|
|
85
|
+
@type uri_parser
|
|
86
|
+
key_name url
|
|
87
|
+
hash_value_field parsed
|
|
88
|
+
out_key_host host
|
|
89
|
+
out_key_path path
|
|
90
|
+
</filter>
|
|
91
|
+
```
|
|
32
92
|
|
|
33
|
-
|
|
93
|
+
```jsonc
|
|
94
|
+
// input
|
|
95
|
+
{ "url": "https://example.com/search" }
|
|
34
96
|
|
|
97
|
+
// output
|
|
98
|
+
{
|
|
99
|
+
"url": "https://example.com/search",
|
|
100
|
+
"parsed": { "host": "example.com", "path": "/search" }
|
|
101
|
+
}
|
|
35
102
|
```
|
|
36
|
-
|
|
103
|
+
|
|
104
|
+
### Namespace output keys — `inject_key_prefix`
|
|
105
|
+
|
|
106
|
+
```aconf
|
|
107
|
+
<filter access.log>
|
|
37
108
|
@type uri_parser
|
|
38
|
-
key_name
|
|
39
|
-
inject_key_prefix
|
|
40
|
-
# hash_value_field parsed
|
|
41
|
-
# suppress_parse_error_log false
|
|
42
|
-
# ignore_key_not_exist false
|
|
43
|
-
# ignore_nil false
|
|
44
|
-
|
|
45
|
-
out_key_scheme scheme
|
|
109
|
+
key_name url
|
|
110
|
+
inject_key_prefix url.
|
|
46
111
|
out_key_host host
|
|
47
|
-
out_key_port port
|
|
48
112
|
out_key_path path
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
113
|
+
</filter>
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
```jsonc
|
|
117
|
+
// input
|
|
118
|
+
{ "url": "https://example.com/search" }
|
|
119
|
+
|
|
120
|
+
// output
|
|
121
|
+
{
|
|
122
|
+
"url": "https://example.com/search",
|
|
123
|
+
"url.host": "example.com",
|
|
124
|
+
"url.path": "/search"
|
|
125
|
+
}
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### Drop empty components — `ignore_nil`
|
|
129
|
+
|
|
130
|
+
When a component is missing (no query, no fragment, etc.) the default is to emit it as `null`. Set `ignore_nil true` to omit those keys entirely.
|
|
131
|
+
|
|
132
|
+
```jsonc
|
|
133
|
+
// input
|
|
134
|
+
{ "url": "https://example.com/path" }
|
|
54
135
|
|
|
55
|
-
|
|
136
|
+
// ignore_nil false (default)
|
|
137
|
+
{ "scheme": "https", "host": "example.com", "port": 443,
|
|
138
|
+
"path": "/path", "query": null, "fragment": null }
|
|
139
|
+
|
|
140
|
+
// ignore_nil true
|
|
141
|
+
{ "scheme": "https", "host": "example.com", "port": 443,
|
|
142
|
+
"path": "/path" }
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
---
|
|
146
|
+
|
|
147
|
+
## `query_string_parser`
|
|
148
|
+
|
|
149
|
+
Decomposes a query string field into individual parameters.
|
|
150
|
+
|
|
151
|
+
> Pairs with an empty key (e.g. the leading `&` in `&foo=1`) are silently dropped — they're noise from user-supplied URLs and never represent a real parameter.
|
|
152
|
+
|
|
153
|
+
### Minimal example
|
|
154
|
+
|
|
155
|
+
```aconf
|
|
156
|
+
<filter access.log>
|
|
157
|
+
@type query_string_parser
|
|
158
|
+
key_name query
|
|
159
|
+
</filter>
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
```jsonc
|
|
163
|
+
// input
|
|
164
|
+
{ "query": "foo=bar&hoge=fuga" }
|
|
165
|
+
|
|
166
|
+
// output
|
|
167
|
+
{ "query": "foo=bar&hoge=fuga", "foo": "bar", "hoge": "fuga" }
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Group output under a single key — `hash_value_field`
|
|
171
|
+
|
|
172
|
+
```aconf
|
|
173
|
+
<filter access.log>
|
|
56
174
|
@type query_string_parser
|
|
57
|
-
key_name
|
|
58
|
-
hash_value_field
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
175
|
+
key_name query
|
|
176
|
+
hash_value_field params
|
|
177
|
+
</filter>
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
```jsonc
|
|
181
|
+
// input
|
|
182
|
+
{ "query": "foo=bar&hoge=fuga" }
|
|
65
183
|
|
|
184
|
+
// output
|
|
185
|
+
{
|
|
186
|
+
"query": "foo=bar&hoge=fuga",
|
|
187
|
+
"params": { "foo": "bar", "hoge": "fuga" }
|
|
188
|
+
}
|
|
66
189
|
```
|
|
67
190
|
|
|
68
|
-
|
|
191
|
+
### Handle repeated parameters
|
|
69
192
|
|
|
70
|
-
|
|
193
|
+
A request like `?tag=ruby&tag=fluentd` has two `tag` values. By default the last one wins (scalar). You have two ways to keep both:
|
|
71
194
|
|
|
72
|
-
|
|
195
|
+
#### Option A: always arrays — `multi_value_params true`
|
|
73
196
|
|
|
74
|
-
|
|
197
|
+
Every parameter becomes an array, even when it appeared once.
|
|
75
198
|
|
|
76
|
-
|
|
199
|
+
```jsonc
|
|
200
|
+
// input
|
|
201
|
+
{ "query": "tag=ruby&tag=fluentd&lang=ja" }
|
|
77
202
|
|
|
78
|
-
|
|
203
|
+
// output
|
|
204
|
+
{ "tag": ["ruby", "fluentd"], "lang": ["ja"] }
|
|
205
|
+
```
|
|
79
206
|
|
|
80
|
-
|
|
207
|
+
#### Option B: array only for listed names — `multi_value_param_names`
|
|
81
208
|
|
|
82
|
-
|
|
209
|
+
You know `tag` may repeat but `lang` won't. Keep `lang` as a scalar and only wrap `tag` in an array.
|
|
83
210
|
|
|
84
|
-
|
|
211
|
+
```aconf
|
|
212
|
+
<filter access.log>
|
|
213
|
+
@type query_string_parser
|
|
214
|
+
key_name query
|
|
215
|
+
multi_value_param_names tag
|
|
216
|
+
</filter>
|
|
217
|
+
```
|
|
85
218
|
|
|
86
|
-
|
|
219
|
+
```jsonc
|
|
220
|
+
// input
|
|
221
|
+
{ "query": "tag=ruby&tag=fluentd&lang=ja" }
|
|
87
222
|
|
|
88
|
-
|
|
223
|
+
// output
|
|
224
|
+
{ "tag": ["ruby", "fluentd"], "lang": "ja" }
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
> When both are set, `multi_value_params true` wins.
|
|
228
|
+
|
|
229
|
+
---
|
|
89
230
|
|
|
90
|
-
|
|
231
|
+
## Options
|
|
91
232
|
|
|
92
|
-
|
|
233
|
+
Shared between both filters unless noted.
|
|
93
234
|
|
|
94
|
-
|
|
95
|
-
|
|
235
|
+
| Option | Type | Default | What it does |
|
|
236
|
+
| --- | --- | --- | --- |
|
|
237
|
+
| `key_name` | string | — *(required)* | Record key holding the URL or query string to parse. |
|
|
238
|
+
| `hash_value_field` | string | `nil` | If set, all extracted fields are nested under this key. |
|
|
239
|
+
| `inject_key_prefix` | string | `nil` | Prefix prepended to every extracted key. |
|
|
240
|
+
| `ignore_key_not_exist` | bool | `false` | When `key_name` is missing, drop the record instead of passing it through. |
|
|
241
|
+
| `emit_invalid_record_to_error` | bool | `true` | When `key_name` is missing, emit the record to Fluentd's error stream. |
|
|
242
|
+
| `suppress_parse_error_log` | bool | `false` | Silence the warning log when the value fails to parse. |
|
|
243
|
+
| `ignore_nil` | bool | `false` | *(uri_parser only)* Omit output keys whose parsed value is `nil`. |
|
|
244
|
+
| `out_key_scheme` / `out_key_host` / `out_key_port` / `out_key_path` / `out_key_query` / `out_key_fragment` | string | `nil` | *(uri_parser only)* Output key name for each URI component. Components without an `out_key_*` are not emitted. |
|
|
245
|
+
| `multi_value_params` | bool | `false` | *(query_string_parser only)* Emit every parameter as an array. |
|
|
246
|
+
| `multi_value_param_names` | array | `nil` | *(query_string_parser only)* Emit only the listed parameters as arrays. Ignored when `multi_value_params` is `true`. |
|
|
247
|
+
|
|
248
|
+
---
|
|
96
249
|
|
|
97
250
|
## Development
|
|
98
251
|
|
|
99
|
-
|
|
252
|
+
```shell
|
|
253
|
+
bundle install
|
|
254
|
+
bundle exec rake test
|
|
255
|
+
```
|
|
100
256
|
|
|
101
|
-
To install this gem onto your local machine
|
|
257
|
+
To install this gem onto your local machine: `bundle exec rake install`.
|
|
258
|
+
To release: bump the version in the gemspec, then `bundle exec rake release` (tags, pushes, and uploads to [rubygems.org](https://rubygems.org)).
|
|
102
259
|
|
|
103
260
|
## Contributing
|
|
104
261
|
|
|
105
|
-
Bug reports and pull requests are welcome
|
|
262
|
+
Bug reports and pull requests are welcome at <https://github.com/daichirata/fluent-plugin-uri-parser>.
|
|
263
|
+
|
|
264
|
+
## License
|
|
106
265
|
|
|
266
|
+
Apache-2.0
|
|
@@ -4,23 +4,25 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "fluent-plugin-uri-parser"
|
|
7
|
-
spec.version = "0.
|
|
7
|
+
spec.version = "0.4.0"
|
|
8
8
|
spec.authors = ["Daichi HIRATA"]
|
|
9
|
-
spec.email = ["
|
|
9
|
+
spec.email = ["daichirata@gmail.com"]
|
|
10
10
|
spec.license = "Apache-2.0"
|
|
11
11
|
|
|
12
12
|
spec.summary = "This is a Fluentd plugin to parse uri and query string in log messages."
|
|
13
13
|
spec.description = "This is a Fluentd plugin to parse uri and query string in log messages."
|
|
14
14
|
spec.homepage = "https://github.com/daichirata/fluent-plugin-uri-parser"
|
|
15
15
|
|
|
16
|
+
spec.required_ruby_version = ">= 3.2"
|
|
17
|
+
|
|
16
18
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
17
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
18
20
|
spec.require_paths = ["lib"]
|
|
19
21
|
|
|
20
|
-
spec.add_runtime_dependency "fluentd", "
|
|
22
|
+
spec.add_runtime_dependency "fluentd", [">= 1.0", "< 2"]
|
|
21
23
|
spec.add_runtime_dependency "addressable"
|
|
22
24
|
|
|
23
|
-
spec.add_development_dependency "
|
|
24
|
-
spec.add_development_dependency "
|
|
25
|
-
spec.add_development_dependency "
|
|
25
|
+
spec.add_development_dependency "bundler"
|
|
26
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
|
27
|
+
spec.add_development_dependency "test-unit", "~> 3.6"
|
|
26
28
|
end
|
|
@@ -1,53 +1,63 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
begin
|
|
27
|
-
params = URI.decode_www_form(raw_value)
|
|
28
|
-
|
|
29
|
-
unless params.empty?
|
|
30
|
-
|
|
31
|
-
if @multi_value_params
|
|
32
|
-
values = Hash.new {|h,k| h[k] = [] }
|
|
33
|
-
params.each{|pair| values[pair[0]].push(pair[1])}
|
|
34
|
-
else
|
|
35
|
-
values = Hash[params]
|
|
1
|
+
require 'fluent/plugin/filter'
|
|
2
|
+
require 'addressable/uri'
|
|
3
|
+
|
|
4
|
+
module Fluent
|
|
5
|
+
module Plugin
|
|
6
|
+
class QueryStringParserFilter < Filter
|
|
7
|
+
Fluent::Plugin.register_filter("query_string_parser", self)
|
|
8
|
+
|
|
9
|
+
config_param :key_name, :string
|
|
10
|
+
config_param :hash_value_field, :string, default: nil
|
|
11
|
+
config_param :inject_key_prefix, :string, default: nil
|
|
12
|
+
config_param :suppress_parse_error_log, :bool, default: false
|
|
13
|
+
config_param :ignore_key_not_exist, :bool, default: false
|
|
14
|
+
config_param :emit_invalid_record_to_error, :bool, default: true
|
|
15
|
+
config_param :multi_value_params, :bool, default: false
|
|
16
|
+
config_param :multi_value_param_names, :array, default: nil
|
|
17
|
+
|
|
18
|
+
def filter(tag, time, record)
|
|
19
|
+
raw_value = record[@key_name]
|
|
20
|
+
|
|
21
|
+
if raw_value.nil?
|
|
22
|
+
if @emit_invalid_record_to_error
|
|
23
|
+
router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
|
|
36
24
|
end
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
25
|
+
return @ignore_key_not_exist ? nil : record
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
begin
|
|
29
|
+
params = Addressable::URI.form_unencode(raw_value)
|
|
30
|
+
params = params.reject {|k, _| k.nil? || k.empty? }
|
|
31
|
+
|
|
32
|
+
unless params.empty?
|
|
33
|
+
if @multi_value_params
|
|
34
|
+
values = Hash.new {|h,k| h[k] = [] }
|
|
35
|
+
params.each{|pair| values[pair[0]].push(pair[1])}
|
|
36
|
+
elsif @multi_value_param_names
|
|
37
|
+
values = {}
|
|
38
|
+
params.each do |k, v|
|
|
39
|
+
if @multi_value_param_names.include?(k)
|
|
40
|
+
(values[k] ||= []) << v
|
|
41
|
+
else
|
|
42
|
+
values[k] = v
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
else
|
|
46
|
+
values = Hash[params]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
if @inject_key_prefix
|
|
50
|
+
values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
|
|
51
|
+
end
|
|
52
|
+
r = @hash_value_field ? { @hash_value_field => values } : values
|
|
53
|
+
record = record.merge(r)
|
|
40
54
|
end
|
|
41
|
-
|
|
42
|
-
|
|
55
|
+
rescue => e
|
|
56
|
+
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
43
57
|
end
|
|
44
58
|
|
|
45
|
-
|
|
46
|
-
rescue => e
|
|
47
|
-
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
59
|
+
return record
|
|
48
60
|
end
|
|
49
61
|
end
|
|
50
|
-
|
|
51
|
-
new_es
|
|
52
62
|
end
|
|
53
63
|
end
|
|
@@ -1,61 +1,61 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
1
|
+
require 'fluent/plugin/filter'
|
|
2
|
+
require 'addressable/uri'
|
|
3
|
+
|
|
4
|
+
module Fluent
|
|
5
|
+
module Plugin
|
|
6
|
+
class URIParserFilter < Filter
|
|
7
|
+
Fluent::Plugin.register_filter("uri_parser", self)
|
|
8
|
+
|
|
9
|
+
config_param :key_name, :string
|
|
10
|
+
config_param :hash_value_field, :string, default: nil
|
|
11
|
+
config_param :inject_key_prefix, :string, default: nil
|
|
12
|
+
config_param :suppress_parse_error_log, :bool, default: false
|
|
13
|
+
config_param :ignore_key_not_exist, :bool, default: false
|
|
14
|
+
config_param :ignore_nil, :bool, default: false
|
|
15
|
+
config_param :emit_invalid_record_to_error, :bool, default: true
|
|
16
|
+
|
|
17
|
+
config_param :out_key_scheme, :string, default: nil
|
|
18
|
+
config_param :out_key_host, :string, default: nil
|
|
19
|
+
config_param :out_key_port, :string, default: nil
|
|
20
|
+
config_param :out_key_path, :string, default: nil
|
|
21
|
+
config_param :out_key_query, :string, default: nil
|
|
22
|
+
config_param :out_key_fragment, :string, default: nil
|
|
23
|
+
|
|
24
|
+
def filter(tag, time, record)
|
|
25
|
+
raw_value = record[@key_name]
|
|
26
|
+
|
|
27
|
+
if raw_value.nil?
|
|
28
|
+
if @emit_invalid_record_to_error
|
|
29
|
+
router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
|
|
30
|
+
end
|
|
31
|
+
return @ignore_key_not_exist ? nil : record
|
|
32
|
+
end
|
|
32
33
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
34
|
+
begin
|
|
35
|
+
uri = Addressable::URI.parse(raw_value)
|
|
36
|
+
|
|
37
|
+
values = {}
|
|
38
|
+
values[@out_key_scheme] = uri.scheme if @out_key_scheme
|
|
39
|
+
values[@out_key_host] = uri.host if @out_key_host
|
|
40
|
+
values[@out_key_port] = uri.inferred_port if @out_key_port
|
|
41
|
+
values[@out_key_path] = uri.path if @out_key_path
|
|
42
|
+
values[@out_key_query] = uri.query if @out_key_query
|
|
43
|
+
values[@out_key_fragment] = uri.fragment if @out_key_fragment
|
|
44
|
+
values.reject! {|_, v| v.nil? } if @ignore_nil
|
|
45
|
+
|
|
46
|
+
unless values.empty?
|
|
47
|
+
if @inject_key_prefix
|
|
48
|
+
values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
|
|
49
|
+
end
|
|
50
|
+
r = @hash_value_field ? { @hash_value_field => values } : values
|
|
51
|
+
record = record.merge(r)
|
|
48
52
|
end
|
|
49
|
-
|
|
50
|
-
|
|
53
|
+
rescue => e
|
|
54
|
+
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
51
55
|
end
|
|
52
56
|
|
|
53
|
-
|
|
54
|
-
rescue => e
|
|
55
|
-
log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
|
|
57
|
+
return record
|
|
56
58
|
end
|
|
57
59
|
end
|
|
58
|
-
|
|
59
|
-
new_es
|
|
60
60
|
end
|
|
61
61
|
end
|
metadata
CHANGED
|
@@ -1,29 +1,35 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-uri-parser
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daichi HIRATA
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-05-22 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
|
16
16
|
requirements:
|
|
17
|
-
- - "
|
|
17
|
+
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version:
|
|
19
|
+
version: '1.0'
|
|
20
|
+
- - "<"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '2'
|
|
20
23
|
type: :runtime
|
|
21
24
|
prerelease: false
|
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
26
|
requirements:
|
|
24
|
-
- - "
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: '1.0'
|
|
30
|
+
- - "<"
|
|
25
31
|
- !ruby/object:Gem::Version
|
|
26
|
-
version:
|
|
32
|
+
version: '2'
|
|
27
33
|
- !ruby/object:Gem::Dependency
|
|
28
34
|
name: addressable
|
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -39,7 +45,7 @@ dependencies:
|
|
|
39
45
|
- !ruby/object:Gem::Version
|
|
40
46
|
version: '0'
|
|
41
47
|
- !ruby/object:Gem::Dependency
|
|
42
|
-
name:
|
|
48
|
+
name: bundler
|
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
|
44
50
|
requirements:
|
|
45
51
|
- - ">="
|
|
@@ -53,40 +59,41 @@ dependencies:
|
|
|
53
59
|
- !ruby/object:Gem::Version
|
|
54
60
|
version: '0'
|
|
55
61
|
- !ruby/object:Gem::Dependency
|
|
56
|
-
name:
|
|
62
|
+
name: rake
|
|
57
63
|
requirement: !ruby/object:Gem::Requirement
|
|
58
64
|
requirements:
|
|
59
65
|
- - "~>"
|
|
60
66
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '
|
|
67
|
+
version: '13.0'
|
|
62
68
|
type: :development
|
|
63
69
|
prerelease: false
|
|
64
70
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
71
|
requirements:
|
|
66
72
|
- - "~>"
|
|
67
73
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: '
|
|
74
|
+
version: '13.0'
|
|
69
75
|
- !ruby/object:Gem::Dependency
|
|
70
|
-
name:
|
|
76
|
+
name: test-unit
|
|
71
77
|
requirement: !ruby/object:Gem::Requirement
|
|
72
78
|
requirements:
|
|
73
79
|
- - "~>"
|
|
74
80
|
- !ruby/object:Gem::Version
|
|
75
|
-
version: '
|
|
81
|
+
version: '3.6'
|
|
76
82
|
type: :development
|
|
77
83
|
prerelease: false
|
|
78
84
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
85
|
requirements:
|
|
80
86
|
- - "~>"
|
|
81
87
|
- !ruby/object:Gem::Version
|
|
82
|
-
version: '
|
|
88
|
+
version: '3.6'
|
|
83
89
|
description: This is a Fluentd plugin to parse uri and query string in log messages.
|
|
84
90
|
email:
|
|
85
|
-
-
|
|
91
|
+
- daichirata@gmail.com
|
|
86
92
|
executables: []
|
|
87
93
|
extensions: []
|
|
88
94
|
extra_rdoc_files: []
|
|
89
95
|
files:
|
|
96
|
+
- ".github/workflows/test.yml"
|
|
90
97
|
- ".gitignore"
|
|
91
98
|
- Gemfile
|
|
92
99
|
- LICENSE.txt
|
|
@@ -95,12 +102,11 @@ files:
|
|
|
95
102
|
- fluent-plugin-uri-parser.gemspec
|
|
96
103
|
- lib/fluent/plugin/filter_query_string_parser.rb
|
|
97
104
|
- lib/fluent/plugin/filter_uri_parser.rb
|
|
98
|
-
- wercker.yml
|
|
99
105
|
homepage: https://github.com/daichirata/fluent-plugin-uri-parser
|
|
100
106
|
licenses:
|
|
101
107
|
- Apache-2.0
|
|
102
108
|
metadata: {}
|
|
103
|
-
post_install_message:
|
|
109
|
+
post_install_message:
|
|
104
110
|
rdoc_options: []
|
|
105
111
|
require_paths:
|
|
106
112
|
- lib
|
|
@@ -108,16 +114,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
108
114
|
requirements:
|
|
109
115
|
- - ">="
|
|
110
116
|
- !ruby/object:Gem::Version
|
|
111
|
-
version: '
|
|
117
|
+
version: '3.2'
|
|
112
118
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
113
119
|
requirements:
|
|
114
120
|
- - ">="
|
|
115
121
|
- !ruby/object:Gem::Version
|
|
116
122
|
version: '0'
|
|
117
123
|
requirements: []
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
signing_key:
|
|
124
|
+
rubygems_version: 3.5.16
|
|
125
|
+
signing_key:
|
|
121
126
|
specification_version: 4
|
|
122
127
|
summary: This is a Fluentd plugin to parse uri and query string in log messages.
|
|
123
128
|
test_files: []
|
data/wercker.yml
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
box: ruby:2.3.0
|
|
2
|
-
# Build definition
|
|
3
|
-
build:
|
|
4
|
-
steps:
|
|
5
|
-
- script:
|
|
6
|
-
name: update bundler
|
|
7
|
-
code: gem update bundler
|
|
8
|
-
|
|
9
|
-
# A step that executes `bundle install` command
|
|
10
|
-
- bundle-install:
|
|
11
|
-
jobs: 4
|
|
12
|
-
|
|
13
|
-
# A custom script step, name value is used in the UI
|
|
14
|
-
# and the code value contains the command that get executed
|
|
15
|
-
- script:
|
|
16
|
-
name: echo ruby information
|
|
17
|
-
code: |
|
|
18
|
-
echo "ruby version $(ruby --version) running"
|
|
19
|
-
echo "from location $(which ruby)"
|
|
20
|
-
echo -p "gem list: $(gem list)"
|
|
21
|
-
|
|
22
|
-
# Add more steps here:
|
|
23
|
-
- script:
|
|
24
|
-
name: test
|
|
25
|
-
code: bundle exec rake test
|