fluent-plugin-jq 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +43 -2
- data/example/Dockerfile +1 -0
- data/example/fluent.conf +18 -0
- data/example/statsd.sample +4 -0
- data/fluent-plugin-jq.gemspec +1 -1
- data/lib/fluent/plugin/parser_jq.rb +56 -0
- data/test/plugin/test_parser_jq.rb +44 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6fef74bd93979266e240f96325d7686b004acb091e16264713e5ccb4431a632d
|
4
|
+
data.tar.gz: f7cc5b63c3863efd09a472a4d0f48c62e3879df6f9eb1ab095bee309f7dfe7e0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3e6335fd5b5099d5165182a28b889a7d7df007466f7c84ac1da31a0c709be0ff4a55baef6125b1351162a3174365140b22a47c110d42ba5263953693eced5179
|
7
|
+
data.tar.gz: df42a0cec7562e2a5882666923169a6871ea046c8c79a28ae9f45cd089f62f30498837948ca9801f162618c53daa5c6d741626642f6a858f3f9e1cdd0d6f555e
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -3,10 +3,11 @@
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-jq.svg)](https://badge.fury.io/rb/fluent-plugin-jq)
|
4
4
|
[![Build Status](https://travis-ci.org/Gimi/fluent-plugin-jq.svg?branch=master)](https://travis-ci.org/Gimi/fluent-plugin-jq)
|
5
5
|
|
6
|
-
A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains
|
6
|
+
A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains four plugins:
|
7
7
|
* `jq` formatter - a formatter plugin formats inputs using jq filters.
|
8
8
|
* `jq_transformer` - a filter plugin transform inputs.
|
9
9
|
* `jq` output - a output plugin uses jq filter to generate new events.
|
10
|
+
* `jq` parser - a parser plugin uses jq filter to parse inputs.
|
10
11
|
|
11
12
|
## Installation
|
12
13
|
|
@@ -137,7 +138,7 @@ It generates two new events:
|
|
137
138
|
|
138
139
|
```javascript
|
139
140
|
{"key": "logLevel", "value": "info"}
|
140
|
-
{"key": log", "value": "this is an example."}
|
141
|
+
{"key": "log", "value": "this is an example."}
|
141
142
|
```
|
142
143
|
|
143
144
|
#### Parameters
|
@@ -157,6 +158,46 @@ Example: If `remove_tag_prefix` is set to 'foo', the input tag foo.bar.baz is tr
|
|
157
158
|
|
158
159
|
Default value: `""`.
|
159
160
|
|
161
|
+
### `jq` Parser
|
162
|
+
|
163
|
+
#### Example
|
164
|
+
|
165
|
+
```
|
166
|
+
<source>
|
167
|
+
@type tail
|
168
|
+
tag tail.*
|
169
|
+
path /some/path/*
|
170
|
+
<parse>
|
171
|
+
@type jq
|
172
|
+
jq 'split(",") | reduce .[] as $item ({}; ($item | split(":")) as $pair | .[$pair[0]] = ($pair[1][:-2] | tonumber))'
|
173
|
+
</parse>
|
174
|
+
</source>
|
175
|
+
```
|
176
|
+
|
177
|
+
Given inputs like
|
178
|
+
|
179
|
+
```
|
180
|
+
cpu.usage:10|g,cpu.free:90|g
|
181
|
+
memory.usage:100|g,memory.rss:80|g
|
182
|
+
```
|
183
|
+
|
184
|
+
It generates records:
|
185
|
+
|
186
|
+
```javascript
|
187
|
+
{"cpu.usage": 10, "cpu.free": 90}
|
188
|
+
{"memory.usage": 100, "memory.rss": 80}
|
189
|
+
```
|
190
|
+
|
191
|
+
#### Parameters
|
192
|
+
|
193
|
+
##### @type (string) (required)
|
194
|
+
|
195
|
+
This must be `jq`.
|
196
|
+
|
197
|
+
##### jq (string) (required)
|
198
|
+
|
199
|
+
The jq filter used to parse inputs. The result of the filter must return an object, otherwise the result will be dropped.
|
200
|
+
|
160
201
|
### Built-in Example
|
161
202
|
|
162
203
|
Once you clone the project from github, you can run the following commands to see a real example for the plugins.
|
data/example/Dockerfile
CHANGED
data/example/fluent.conf
CHANGED
@@ -4,6 +4,17 @@
|
|
4
4
|
dummy {"log": "everything goes well", "stream": "stdout"}
|
5
5
|
</source>
|
6
6
|
|
7
|
+
<source>
|
8
|
+
@type tail
|
9
|
+
tag stats
|
10
|
+
path /statsd.sample
|
11
|
+
read_from_head true
|
12
|
+
<parse>
|
13
|
+
@type jq
|
14
|
+
jq 'split(",") | reduce .[] as $item ({}; ($item | split(":")) as $pair | .[$pair[0]] = ($pair[1][:-2] | tonumber))'
|
15
|
+
</parse>
|
16
|
+
</source>
|
17
|
+
|
7
18
|
<match raw.dummy>
|
8
19
|
@type jq
|
9
20
|
jq .record | to_entries
|
@@ -22,3 +33,10 @@
|
|
22
33
|
jq '"\(.time | todate) [\(.tag)] \(.key) => \(.value)"'
|
23
34
|
</format>
|
24
35
|
</match>
|
36
|
+
|
37
|
+
<match stats>
|
38
|
+
@type stdout
|
39
|
+
<format>
|
40
|
+
@type json
|
41
|
+
</format>
|
42
|
+
</match>
|
data/fluent-plugin-jq.gemspec
CHANGED
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
require "fluent/plugin/parser"
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Plugin
|
21
|
+
class JqParser < Fluent::Plugin::Parser
|
22
|
+
Fluent::Plugin.register_parser("jq", self)
|
23
|
+
|
24
|
+
desc 'The jq filter used to format the input. The result of the filter must return an object.'
|
25
|
+
config_param :jq, :string
|
26
|
+
|
27
|
+
def initialize
|
28
|
+
super
|
29
|
+
require "jq"
|
30
|
+
end
|
31
|
+
|
32
|
+
def configure(conf)
|
33
|
+
super
|
34
|
+
@jq_filter = JQ::Core.new @jq
|
35
|
+
rescue JQ::Error
|
36
|
+
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
|
37
|
+
end
|
38
|
+
|
39
|
+
def parse(text)
|
40
|
+
record = [].tap { |buf|
|
41
|
+
@jq_filter.update(MultiJson.dump(text), false) { |r|
|
42
|
+
buf << MultiJson.load("[#{r}]").first
|
43
|
+
}
|
44
|
+
}.first
|
45
|
+
if record.is_a?(Hash)
|
46
|
+
yield parse_time(record), record
|
47
|
+
else
|
48
|
+
log.error "jq filter #{@jq} did not return a hash, skip this record."
|
49
|
+
end
|
50
|
+
rescue JQ::Error
|
51
|
+
log.error "Failed to parse #{text} with #{@jq}, error: #{$!.message}"
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
require "fluent/test/driver/parser"
|
5
|
+
require "fluent/plugin/parser_jq"
|
6
|
+
|
7
|
+
class JqParserTest < Test::Unit::TestCase
|
8
|
+
setup do
|
9
|
+
Fluent::Test.setup
|
10
|
+
end
|
11
|
+
|
12
|
+
test "it should require jq" do
|
13
|
+
assert_raise(Fluent::ConfigError) { create_driver '' }
|
14
|
+
end
|
15
|
+
|
16
|
+
test "it should raise error on invalid jq program" do
|
17
|
+
e = assert_raise(Fluent::ConfigError) { create_driver 'jq blah' }
|
18
|
+
assert_match(/compile error/, e.message)
|
19
|
+
end
|
20
|
+
|
21
|
+
test "it should work" do
|
22
|
+
d = create_driver 'jq split(",") | reduce .[] as $item ({}; ($item | split(":")) as $pair | .[$pair[0]] = $pair[1])'
|
23
|
+
text = "name:jq,type:parser,author:Gimi"
|
24
|
+
expected_record = {"name" => "jq", "type" => "parser", "author" => "Gimi"}
|
25
|
+
d.instance.parse(text) { |time, record|
|
26
|
+
assert_equal expected_record, record
|
27
|
+
}
|
28
|
+
assert_nil d.instance.log.logs.find { |log| log =~ /\[error\]/ }
|
29
|
+
end
|
30
|
+
|
31
|
+
test "it should skip if it does not return a hash" do
|
32
|
+
d = create_driver 'jq split(",")'
|
33
|
+
text = "name:jq,type:parser,author:Gimi"
|
34
|
+
expected_record = {"name": "jq", "type": "parser", "author": "Gimi"}
|
35
|
+
d.instance.parse(text) { |time, record| assert false }
|
36
|
+
assert d.instance.log.logs.any? { |log| log =~ /\[error\]/ }
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def create_driver(conf)
|
42
|
+
Fluent::Test::Driver::Parser.new(Fluent::Plugin::JqParser).configure(conf)
|
43
|
+
end
|
44
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-jq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zhimin (Gimi) Liang
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-03-
|
11
|
+
date: 2018-03-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -131,15 +131,18 @@ files:
|
|
131
131
|
- Rakefile
|
132
132
|
- example/Dockerfile
|
133
133
|
- example/fluent.conf
|
134
|
+
- example/statsd.sample
|
134
135
|
- fluent-plugin-jq.gemspec
|
135
136
|
- lib/fluent/plugin/filter_jq_transformer.rb
|
136
137
|
- lib/fluent/plugin/formatter_jq.rb
|
137
138
|
- lib/fluent/plugin/out_jq.rb
|
139
|
+
- lib/fluent/plugin/parser_jq.rb
|
138
140
|
- run_ci.sh
|
139
141
|
- test/helper.rb
|
140
142
|
- test/plugin/test_filter_jq_transformer.rb
|
141
143
|
- test/plugin/test_formatter_jq.rb
|
142
144
|
- test/plugin/test_out_jq.rb
|
145
|
+
- test/plugin/test_parser_jq.rb
|
143
146
|
homepage: https://github.com/Gimi/fluent-plugin-jq
|
144
147
|
licenses:
|
145
148
|
- Apache-2.0
|
@@ -167,5 +170,6 @@ summary: Fluentd plugins uses the jq engine.
|
|
167
170
|
test_files:
|
168
171
|
- test/helper.rb
|
169
172
|
- test/plugin/test_filter_jq_transformer.rb
|
173
|
+
- test/plugin/test_parser_jq.rb
|
170
174
|
- test/plugin/test_out_jq.rb
|
171
175
|
- test/plugin/test_formatter_jq.rb
|