fluent-plugin-jq 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +44 -1
- data/example/fluent.conf +8 -2
- data/fluent-plugin-jq.gemspec +1 -1
- data/lib/fluent/plugin/filter_jq_transformer.rb +6 -2
- data/lib/fluent/plugin/formatter_jq.rb +6 -2
- data/lib/fluent/plugin/out_jq.rb +65 -0
- data/test/plugin/test_out_jq.rb +58 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 85add62e095656dfaeb43bff27ae998d7ff39cd523a2f0d2d564d981a8e3ccf6
|
4
|
+
data.tar.gz: ec02443a917e479f171ecc44603f39b4f3c7f5e3ecbe629c0a2c4ce8eabe2bc4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4f340a9588d62e49758f7e8331768d912b4518876ef97945a82f9a6977fd516b23a32ece0712371f42bbd29607961e1ceb9ce2b0ce95daf8ba832d3a727ba247
|
7
|
+
data.tar.gz: 0601f3c89a55354fd844c6ef0152879d020a0af3c4d9980c9e5109622f25affd54ace34fe351f44e50eb825af137861721d90c015c74b19f97847c44b5341298
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -3,9 +3,10 @@
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/fluent-plugin-jq.svg)](https://badge.fury.io/rb/fluent-plugin-jq)
|
4
4
|
[![Build Status](https://travis-ci.org/Gimi/fluent-plugin-jq.svg?branch=master)](https://travis-ci.org/Gimi/fluent-plugin-jq)
|
5
5
|
|
6
|
-
A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains
|
6
|
+
A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedolan.github.io/jq/). It now contains three plugins:
|
7
7
|
* `jq` formatter - a formatter plugin formats inputs using jq filters.
|
8
8
|
* `jq_transformer` - a filter plugin transform inputs.
|
9
|
+
* `jq` output - a output plugin uses jq filter to generate new events.
|
9
10
|
|
10
11
|
## Installation
|
11
12
|
|
@@ -114,6 +115,48 @@ This must be `jq_transformer`.
|
|
114
115
|
|
115
116
|
The jq filter used to transform the input. The result of the filter should return an object. If after applying the transforming the new event is not an object (a hash), the event will be dropped.
|
116
117
|
|
118
|
+
### `jq` Output
|
119
|
+
|
120
|
+
#### Example
|
121
|
+
|
122
|
+
```
|
123
|
+
<match raw.data>
|
124
|
+
@type jq
|
125
|
+
jq .record | to_entries
|
126
|
+
remove_tag_prefix raw
|
127
|
+
</filter>
|
128
|
+
```
|
129
|
+
|
130
|
+
The above example will generate one event for each key-value pair in each input, and then tag it with "data" ("raw." is removed), and send it back to router. For example, given an input like
|
131
|
+
|
132
|
+
```javascript
|
133
|
+
{"logLevel": "info", "log": "this is an example."}
|
134
|
+
```
|
135
|
+
|
136
|
+
It generates two new events:
|
137
|
+
|
138
|
+
```javascript
|
139
|
+
{"key": "logLevel", "value": "info"}
|
140
|
+
{"key": log", "value": "this is an example."}
|
141
|
+
```
|
142
|
+
|
143
|
+
#### Parameters
|
144
|
+
|
145
|
+
##### @type (string) (required)
|
146
|
+
|
147
|
+
This must be `jq`.
|
148
|
+
|
149
|
+
##### jq (string) (required)
|
150
|
+
|
151
|
+
The jq filter used to generate new events. The result of the filter should return an object or an array of objects. New events will be put back to router so that they will be processed again.
|
152
|
+
|
153
|
+
##### remove_tag_prefix (string) (optional)
|
154
|
+
|
155
|
+
The prefix to remove from the input tag when outputting a new event. A prefix has to be a complete tag part.
|
156
|
+
Example: If `remove_tag_prefix` is set to 'foo', the input tag foo.bar.baz is transformed to bar.baz and the input tag 'foofoo.bar' is not modified.
|
157
|
+
|
158
|
+
Default value: `""`.
|
159
|
+
|
117
160
|
### Built-in Example
|
118
161
|
|
119
162
|
Once you clone the project from github, you can run the following commands to see a real example for the plugins.
|
data/example/fluent.conf
CHANGED
@@ -1,9 +1,15 @@
|
|
1
1
|
<source>
|
2
2
|
@type dummy
|
3
|
-
tag dummy
|
3
|
+
tag raw.dummy
|
4
4
|
dummy {"log": "everything goes well", "stream": "stdout"}
|
5
5
|
</source>
|
6
6
|
|
7
|
+
<match raw.dummy>
|
8
|
+
@type jq
|
9
|
+
jq .record | to_entries
|
10
|
+
remove_tag_prefix raw
|
11
|
+
</match>
|
12
|
+
|
7
13
|
<filter dummy>
|
8
14
|
@type jq_transformer
|
9
15
|
jq .record + {tag, time}
|
@@ -13,6 +19,6 @@
|
|
13
19
|
@type stdout
|
14
20
|
<format>
|
15
21
|
@type jq
|
16
|
-
jq '"\(.time | todate) [\(.tag)] \(.
|
22
|
+
jq '"\(.time | todate) [\(.tag)] \(.key) => \(.value)"'
|
17
23
|
</format>
|
18
24
|
</match>
|
data/fluent-plugin-jq.gemspec
CHANGED
@@ -31,13 +31,17 @@ module Fluent
|
|
31
31
|
|
32
32
|
def configure(conf)
|
33
33
|
super
|
34
|
-
JQ::Core.new @jq
|
34
|
+
@jq_filter = JQ::Core.new @jq
|
35
35
|
rescue JQ::Error
|
36
36
|
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
|
37
37
|
end
|
38
38
|
|
39
39
|
def filter(tag, time, record)
|
40
|
-
new_record =
|
40
|
+
new_record = [].tap { |buf|
|
41
|
+
@jq_filter.update(MultiJson.dump(tag: tag, time: time, record: record), false) { |r|
|
42
|
+
buf << MultiJson.load("[#{r}]").first
|
43
|
+
}
|
44
|
+
}.first
|
41
45
|
return new_record if new_record.is_a?(Hash)
|
42
46
|
|
43
47
|
log.error "jq filter #{@jq} did not return a hash, skip this record."
|
@@ -41,13 +41,17 @@ module Fluent
|
|
41
41
|
@jq = @jq_program unless @jq
|
42
42
|
raise Fluent::ConfigError, "jq is required." unless @jq
|
43
43
|
|
44
|
-
JQ::Core.new @jq
|
44
|
+
@jq_filter = JQ::Core.new @jq
|
45
45
|
rescue JQ::Error
|
46
46
|
raise Fluent::ConfigError, "Could not parse jq filter #{@jq}, error: #{$!.message}"
|
47
47
|
end
|
48
48
|
|
49
49
|
def format(tag, time, record)
|
50
|
-
item =
|
50
|
+
item = [].tap { |buf|
|
51
|
+
@jq_filter.update(MultiJson.dump(record), false) { |r|
|
52
|
+
buf << MultiJson.load("[#{r}]").first
|
53
|
+
}
|
54
|
+
}.first
|
51
55
|
return item if item.instance_of?(String)
|
52
56
|
MultiJson.dump item
|
53
57
|
rescue JQ::Error
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Copyright 2018- Zhimin (Gimi) Liang (https://github.com/Gimi)
|
4
|
+
#
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
# you may not use this file except in compliance with the License.
|
7
|
+
# You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
# See the License for the specific language governing permissions and
|
15
|
+
# limitations under the License.
|
16
|
+
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
|
19
|
+
module Fluent::Plugin
|
20
|
+
class JqOutput < Output
|
21
|
+
Fluent::Plugin.register_output('jq', self)
|
22
|
+
helpers :event_emitter
|
23
|
+
|
24
|
+
desc 'The jq filter used to transform the input. The result of the filter should return an object.'
|
25
|
+
config_param :jq, :string
|
26
|
+
|
27
|
+
desc 'The prefix to be removed from the input tag when outputting a new record.'
|
28
|
+
config_param :remove_tag_prefix, :string, default: ''
|
29
|
+
|
30
|
+
def initialize
|
31
|
+
super
|
32
|
+
require "jq"
|
33
|
+
end
|
34
|
+
|
35
|
+
def configure(conf)
|
36
|
+
super
|
37
|
+
@jq_filter = JQ::Core.new @jq
|
38
|
+
rescue JQ::Error
|
39
|
+
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
|
40
|
+
end
|
41
|
+
|
42
|
+
def multi_workers_ready?
|
43
|
+
true
|
44
|
+
end
|
45
|
+
|
46
|
+
def process(tag, es)
|
47
|
+
new_es = Fluent::MultiEventStream.new
|
48
|
+
es.each do |time, record|
|
49
|
+
begin
|
50
|
+
@jq_filter.update(MultiJson.dump(tag: tag, time: time, record: record), false) { |r|
|
51
|
+
# the filter could return an array
|
52
|
+
new_records = [MultiJson.load("[#{r}]").first]
|
53
|
+
new_records.flatten!
|
54
|
+
new_records.each { |new_record| new_es.add time, new_record }
|
55
|
+
}
|
56
|
+
rescue JQ::Error
|
57
|
+
log.error "Failed to transform #{MultiJson.dump record} with #{@jq}, error: #{$!.message}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
new_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
|
62
|
+
router.emit_stream(new_tag, new_es)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
require "fluent/test/driver/output"
|
5
|
+
require "fluent/plugin/out_jq"
|
6
|
+
|
7
|
+
class JqOutputTest < Test::Unit::TestCase
|
8
|
+
setup do
|
9
|
+
Fluent::Test.setup
|
10
|
+
end
|
11
|
+
|
12
|
+
test "it should require jq" do
|
13
|
+
assert_raise(Fluent::ConfigError) { create_driver '' }
|
14
|
+
end
|
15
|
+
|
16
|
+
test "it should raise error on invalid jq program" do
|
17
|
+
e = assert_raise(Fluent::ConfigError) { create_driver 'jq blah' }
|
18
|
+
assert_match(/compile error/, e.message)
|
19
|
+
end
|
20
|
+
|
21
|
+
test "it should work on tag" do
|
22
|
+
events = create_driver('jq "{tag}"').events
|
23
|
+
assert_equal events.size, 1
|
24
|
+
assert_equal events[0][2]["tag"], "some.tag"
|
25
|
+
end
|
26
|
+
|
27
|
+
test "it should work on time" do
|
28
|
+
events = create_driver('jq "{time: .time | gmtime }"').events
|
29
|
+
assert_equal events.size, 1
|
30
|
+
assert_equal events[0][2]["time"][0..5], [2018, 2, 22, 1, 23, 45]
|
31
|
+
end
|
32
|
+
|
33
|
+
test "it should support array" do
|
34
|
+
events = create_driver('jq ".record | to_entries"').events.map { |evt| evt[2] }
|
35
|
+
assert_equal events.size, 2
|
36
|
+
assert_include events, {"key" => "log", "value" => "this is a log"}
|
37
|
+
assert_include events, {"key" => "source", "value" => "stdout"}
|
38
|
+
end
|
39
|
+
|
40
|
+
test "it should remove specified tag prefix" do
|
41
|
+
events = create_driver(<<~CONF).events
|
42
|
+
jq ".record"
|
43
|
+
remove_tag_prefix some
|
44
|
+
CONF
|
45
|
+
|
46
|
+
assert_equal events.size, 1
|
47
|
+
assert_equal events[0][0], "tag"
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def create_driver(conf)
|
53
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::JqOutput).configure(conf).tap { |d|
|
54
|
+
time = event_time("2018-03-22 01:23:45 UTC")
|
55
|
+
d.run { d.feed "some.tag", time, {"log" => "this is a log", "source" => "stdout"} }
|
56
|
+
}
|
57
|
+
end
|
58
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-jq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zhimin (Gimi) Liang
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-03-
|
11
|
+
date: 2018-03-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -134,10 +134,12 @@ files:
|
|
134
134
|
- fluent-plugin-jq.gemspec
|
135
135
|
- lib/fluent/plugin/filter_jq_transformer.rb
|
136
136
|
- lib/fluent/plugin/formatter_jq.rb
|
137
|
+
- lib/fluent/plugin/out_jq.rb
|
137
138
|
- run_ci.sh
|
138
139
|
- test/helper.rb
|
139
140
|
- test/plugin/test_filter_jq_transformer.rb
|
140
141
|
- test/plugin/test_formatter_jq.rb
|
142
|
+
- test/plugin/test_out_jq.rb
|
141
143
|
homepage: https://github.com/Gimi/fluent-plugin-jq
|
142
144
|
licenses:
|
143
145
|
- Apache-2.0
|
@@ -165,4 +167,5 @@ summary: Fluentd plugins uses the jq engine.
|
|
165
167
|
test_files:
|
166
168
|
- test/helper.rb
|
167
169
|
- test/plugin/test_filter_jq_transformer.rb
|
170
|
+
- test/plugin/test_out_jq.rb
|
168
171
|
- test/plugin/test_formatter_jq.rb
|