fluent-plugin-jq 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -4
- data/README.md +2 -2
- data/example/Dockerfile +1 -1
- data/fluent-plugin-jq.gemspec +1 -2
- data/lib/fluent/plugin/filter_jq_transformer.rb +6 -20
- data/lib/fluent/plugin/formatter_jq.rb +12 -26
- data/lib/fluent/plugin/jq_mixin.rb +47 -0
- data/lib/fluent/plugin/out_jq.rb +9 -22
- data/lib/fluent/plugin/parser_jq.rb +6 -20
- data/test/plugin/test_filter_jq_transformer.rb +5 -1
- data/test/plugin/test_formatter_jq.rb +5 -1
- data/test/plugin/test_parser_jq.rb +5 -1
- metadata +3 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7bee9a7bb5bd22f8b9a08cc91e3e0b8c82cc26c3a22c79840513c36d18eeda0b
|
4
|
+
data.tar.gz: adaa5a025cff8142f3e34f9a518df08dcab30b593e94193ca046a902ca94dc84
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c7d787d80c6b742ad54e74d6fea4273710f59308b1629cf6064d06a5e4ad397d32bcf97c9de12b5045221555b60537ff859e171c25a5c2c5df15c83e07a04e87
|
7
|
+
data.tar.gz: ef43d2ed72e2c4d0a4f5fd0559cd1e327f3bdb52dcaaa8f9f762a3081dd22642354ba5f74e2eaac5858c2da40ce5a803902e53a489f6503ef9ee9b2f3a054283
|
data/Gemfile.lock
CHANGED
@@ -1,10 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-jq (0.
|
4
|
+
fluent-plugin-jq (0.5.0)
|
5
5
|
fluentd (>= 0.14.10, < 2)
|
6
6
|
multi_json (~> 1.13)
|
7
|
-
ruby-jq (~> 0.1)
|
8
7
|
|
9
8
|
GEM
|
10
9
|
remote: https://rubygems.org/
|
@@ -35,8 +34,6 @@ GEM
|
|
35
34
|
multi_json (1.13.1)
|
36
35
|
power_assert (1.1.1)
|
37
36
|
rake (12.3.0)
|
38
|
-
ruby-jq (0.1.7)
|
39
|
-
multi_json
|
40
37
|
serverengine (2.0.6)
|
41
38
|
sigdump (~> 0.2.2)
|
42
39
|
sigdump (0.2.4)
|
data/README.md
CHANGED
@@ -13,7 +13,7 @@ A collection of [Fluentd](https://fluentd.org/) plugins use [jq](https://stedola
|
|
13
13
|
|
14
14
|
See also: [Plugin Management](https://docs.fluentd.org/v1.0/articles/plugin-management).
|
15
15
|
|
16
|
-
Before you install this plugin, please make sure
|
16
|
+
Before you install this plugin, please make sure the `jq` command line tool has been installed on your machine. Plugins defined in this gem will call the `jq` command to make the transformation.
|
17
17
|
|
18
18
|
### RubyGems
|
19
19
|
|
@@ -73,7 +73,7 @@ This must be `jq`.
|
|
73
73
|
|
74
74
|
##### jq (string) (required)
|
75
75
|
|
76
|
-
The jq filter for formatting income events. The result
|
76
|
+
The jq filter for formatting income events. The returned result should be a string, if not, it will be encoded to a JSON representation in a string.
|
77
77
|
|
78
78
|
##### on_error (enum) (optional)
|
79
79
|
|
data/example/Dockerfile
CHANGED
data/fluent-plugin-jq.gemspec
CHANGED
@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |spec|
|
5
5
|
spec.name = "fluent-plugin-jq"
|
6
|
-
spec.version = "0.
|
6
|
+
spec.version = "0.5.0"
|
7
7
|
spec.authors = ["Zhimin (Gimi) Liang"]
|
8
8
|
spec.email = ["liang.gimi@gmail.com"]
|
9
9
|
|
@@ -24,6 +24,5 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_development_dependency "coveralls", "~> 0.8"
|
25
25
|
|
26
26
|
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
27
|
-
spec.add_runtime_dependency "ruby-jq", "~> 0.1"
|
28
27
|
spec.add_runtime_dependency "multi_json", "~> 1.13"
|
29
28
|
end
|
@@ -15,39 +15,25 @@
|
|
15
15
|
# limitations under the License.
|
16
16
|
|
17
17
|
require "fluent/plugin/filter"
|
18
|
+
require "fluent/plugin/jq_mixin"
|
18
19
|
|
19
20
|
module Fluent
|
20
21
|
module Plugin
|
21
22
|
class JqTransformerFilter < Fluent::Plugin::Filter
|
22
23
|
Fluent::Plugin.register_filter("jq_transformer", self)
|
23
24
|
|
24
|
-
|
25
|
-
config_param :jq, :string
|
25
|
+
include JqMixin
|
26
26
|
|
27
|
-
|
28
|
-
super
|
29
|
-
require "jq"
|
30
|
-
end
|
31
|
-
|
32
|
-
def configure(conf)
|
33
|
-
super
|
34
|
-
@jq_filter = JQ::Core.new @jq
|
35
|
-
rescue JQ::Error
|
36
|
-
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
|
37
|
-
end
|
27
|
+
config_set_desc :jq, 'The jq filter used to transform the input. The result of the filter should return an object.'
|
38
28
|
|
39
29
|
def filter(tag, time, record)
|
40
|
-
new_record =
|
41
|
-
@jq_filter.update(MultiJson.dump(tag: tag, time: time, record: record), false) { |r|
|
42
|
-
buf << MultiJson.load("[#{r}]").first
|
43
|
-
}
|
44
|
-
}.first
|
30
|
+
new_record = jq_transform tag: tag, time: time, record: record
|
45
31
|
return new_record if new_record.is_a?(Hash)
|
46
32
|
|
47
33
|
log.error "jq filter #{@jq} did not return a hash, skip this record."
|
48
34
|
nil
|
49
|
-
rescue
|
50
|
-
log.error "
|
35
|
+
rescue JqError
|
36
|
+
log.error "Filter failed with #{@jq}#{log.on_debug { ' on ' + MultiJson.dump(tag: tag, time: time, record: record) }}, error: #{$!.message}"
|
51
37
|
nil
|
52
38
|
end
|
53
39
|
end
|
@@ -15,53 +15,39 @@
|
|
15
15
|
# limitations under the License.
|
16
16
|
|
17
17
|
require "fluent/plugin/formatter"
|
18
|
+
require "fluent/plugin/jq_mixin"
|
18
19
|
|
19
20
|
module Fluent
|
20
21
|
module Plugin
|
21
22
|
class JqFormatter < Fluent::Plugin::Formatter
|
22
23
|
Fluent::Plugin.register_formatter("jq", self)
|
23
24
|
|
24
|
-
|
25
|
-
config_param :jq, :string, default: nil
|
25
|
+
include JqMixin
|
26
26
|
|
27
|
-
|
28
|
-
config_param :jq_program, :string, deprecated: 'use jq instead.', default: nil
|
27
|
+
config_set_desc :jq, 'The jq filter used to format income events. If the result returned from the filter is not a string, it will be encoded as a JSON string.'
|
29
28
|
|
30
29
|
desc 'Defines the behavior on error happens when formatting an event. "skip" will skip the event; "ignore" will ignore the error and return the JSON representation of the original event; "raise_error" will raise a RuntimeError.'
|
31
30
|
config_param :on_error, :enum, list: [:skip, :ignore, :raise_error], default: :ignore
|
32
31
|
|
33
32
|
def initialize
|
34
33
|
super
|
35
|
-
require "jq"
|
36
|
-
end
|
37
|
-
|
38
|
-
def configure(conf)
|
39
|
-
super
|
40
|
-
|
41
|
-
@jq = @jq_program unless @jq
|
42
|
-
raise Fluent::ConfigError, "jq is required." unless @jq
|
43
|
-
|
44
|
-
@jq_filter = JQ::Core.new @jq
|
45
|
-
rescue JQ::Error
|
46
|
-
raise Fluent::ConfigError, "Could not parse jq filter #{@jq}, error: #{$!.message}"
|
47
34
|
end
|
48
35
|
|
49
36
|
def format(tag, time, record)
|
50
|
-
item =
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
msg = "Failed to format #{record.to_json} with #{@jq}, error: #{$!.message}"
|
37
|
+
item = jq_transform record
|
38
|
+
if item.instance_of?(String)
|
39
|
+
item
|
40
|
+
else
|
41
|
+
MultiJson.dump item
|
42
|
+
end
|
43
|
+
rescue JqError
|
44
|
+
msg = "Format failed with #{@jq}#{log.on_debug { ' on ' + MultiJson.dump(record) }}, error: #{$!.message}"
|
59
45
|
log.error msg
|
60
46
|
case @on_error
|
61
47
|
when :skip
|
62
48
|
return ''
|
63
49
|
when :ignore
|
64
|
-
return record
|
50
|
+
return MultiJson.dump(record)
|
65
51
|
when :raise_error
|
66
52
|
raise msg
|
67
53
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'shellwords'
|
2
|
+
require 'multi_json'
|
3
|
+
|
4
|
+
module JqMixin
|
5
|
+
JqError = Class.new(RuntimeError)
|
6
|
+
|
7
|
+
def self.included(plugin)
|
8
|
+
plugin.config_param :jq, :string
|
9
|
+
end
|
10
|
+
|
11
|
+
def configure(conf)
|
12
|
+
super
|
13
|
+
p = start_process(null_input: true)
|
14
|
+
err = p.read
|
15
|
+
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{err}" if err =~ /compile error/m
|
16
|
+
rescue
|
17
|
+
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
|
18
|
+
ensure
|
19
|
+
p.close if p # if `super` fails, `p` will be `nil`
|
20
|
+
end
|
21
|
+
|
22
|
+
def start
|
23
|
+
super
|
24
|
+
@jq_process = start_process
|
25
|
+
end
|
26
|
+
|
27
|
+
def shutdown
|
28
|
+
@jq_process.close rescue nil
|
29
|
+
super
|
30
|
+
end
|
31
|
+
|
32
|
+
def start_process(filter: @jq, null_input: false)
|
33
|
+
IO.popen(%Q"jq #{'-n' if null_input} --unbuffered -c '#{filter}' 2>&1", 'r+')
|
34
|
+
end
|
35
|
+
|
36
|
+
def jq_transform(object)
|
37
|
+
@jq_process.puts MultiJson.dump(object)
|
38
|
+
result = @jq_process.gets
|
39
|
+
MultiJson.load result
|
40
|
+
rescue MultiJson::ParseError
|
41
|
+
raise JqError.new(result)
|
42
|
+
rescue Errno::EPIPE
|
43
|
+
@jq_process.close
|
44
|
+
@jq_process = start_process
|
45
|
+
retry
|
46
|
+
end
|
47
|
+
end
|
data/lib/fluent/plugin/out_jq.rb
CHANGED
@@ -15,30 +15,20 @@
|
|
15
15
|
# limitations under the License.
|
16
16
|
|
17
17
|
require 'fluent/plugin/output'
|
18
|
+
require 'fluent/plugin/jq_mixin'
|
18
19
|
|
19
20
|
module Fluent::Plugin
|
20
21
|
class JqOutput < Output
|
21
22
|
Fluent::Plugin.register_output('jq', self)
|
22
23
|
helpers :event_emitter
|
23
24
|
|
24
|
-
|
25
|
-
|
25
|
+
include JqMixin
|
26
|
+
|
27
|
+
config_set_desc :jq, 'The jq filter used to transform the input. If the filter returns an array, each object in the array will be a new record.'
|
26
28
|
|
27
29
|
desc 'The prefix to be removed from the input tag when outputting a new record.'
|
28
30
|
config_param :remove_tag_prefix, :string, default: ''
|
29
31
|
|
30
|
-
def initialize
|
31
|
-
super
|
32
|
-
require "jq"
|
33
|
-
end
|
34
|
-
|
35
|
-
def configure(conf)
|
36
|
-
super
|
37
|
-
@jq_filter = JQ::Core.new @jq
|
38
|
-
rescue JQ::Error
|
39
|
-
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
|
40
|
-
end
|
41
|
-
|
42
32
|
def multi_workers_ready?
|
43
33
|
true
|
44
34
|
end
|
@@ -47,14 +37,11 @@ module Fluent::Plugin
|
|
47
37
|
new_es = Fluent::MultiEventStream.new
|
48
38
|
es.each do |time, record|
|
49
39
|
begin
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
}
|
56
|
-
rescue JQ::Error
|
57
|
-
log.error "Failed to transform #{MultiJson.dump record} with #{@jq}, error: #{$!.message}"
|
40
|
+
new_records = jq_transform tag: tag, time: time, record: record
|
41
|
+
new_records = [new_records] unless new_records.is_a?(Array)
|
42
|
+
new_records.each { |new_record| new_es.add time, new_record }
|
43
|
+
rescue JqError
|
44
|
+
log.error "Process failed with #{@jq}#{log.on_debug {' on ' + MultiJson.dump(record)}}, error: #{$!.message}"
|
58
45
|
end
|
59
46
|
end
|
60
47
|
|
@@ -15,40 +15,26 @@
|
|
15
15
|
# limitations under the License.
|
16
16
|
|
17
17
|
require "fluent/plugin/parser"
|
18
|
+
require 'fluent/plugin/jq_mixin'
|
18
19
|
|
19
20
|
module Fluent
|
20
21
|
module Plugin
|
21
22
|
class JqParser < Fluent::Plugin::Parser
|
22
23
|
Fluent::Plugin.register_parser("jq", self)
|
23
24
|
|
24
|
-
|
25
|
-
config_param :jq, :string
|
25
|
+
include JqMixin
|
26
26
|
|
27
|
-
|
28
|
-
super
|
29
|
-
require "jq"
|
30
|
-
end
|
31
|
-
|
32
|
-
def configure(conf)
|
33
|
-
super
|
34
|
-
@jq_filter = JQ::Core.new @jq
|
35
|
-
rescue JQ::Error
|
36
|
-
raise Fluent::ConfigError, "Could not parse jq filter: #{@jq}, error: #{$!.message}"
|
37
|
-
end
|
27
|
+
config_set_desc :jq, 'The jq filter used to format the input. The result of the filter must return an object.'
|
38
28
|
|
39
29
|
def parse(text)
|
40
|
-
record =
|
41
|
-
@jq_filter.update(MultiJson.dump(text), false) { |r|
|
42
|
-
buf << MultiJson.load("[#{r}]").first
|
43
|
-
}
|
44
|
-
}.first
|
30
|
+
record = jq_transform text
|
45
31
|
if record.is_a?(Hash)
|
46
32
|
yield parse_time(record), record
|
47
33
|
else
|
48
34
|
log.error "jq filter #{@jq} did not return a hash, skip this record."
|
49
35
|
end
|
50
|
-
rescue
|
51
|
-
log.error "
|
36
|
+
rescue JqError
|
37
|
+
log.error "Parse failed with #{@jq}#{log.on_debug {' on ' + text}}, error: #{$!.message}"
|
52
38
|
nil
|
53
39
|
end
|
54
40
|
end
|
@@ -9,6 +9,10 @@ class JqTransformerFilterTest < Test::Unit::TestCase
|
|
9
9
|
Fluent::Test.setup
|
10
10
|
end
|
11
11
|
|
12
|
+
teardown do
|
13
|
+
@driver.instance.shutdown if @driver
|
14
|
+
end
|
15
|
+
|
12
16
|
test "it should require jq" do
|
13
17
|
assert_raise(Fluent::ConfigError) { create_driver '' }
|
14
18
|
end
|
@@ -65,6 +69,6 @@ class JqTransformerFilterTest < Test::Unit::TestCase
|
|
65
69
|
private
|
66
70
|
|
67
71
|
def create_driver(conf)
|
68
|
-
Fluent::Test::Driver::Filter.new(Fluent::Plugin::JqTransformerFilter).configure(conf)
|
72
|
+
@driver = Fluent::Test::Driver::Filter.new(Fluent::Plugin::JqTransformerFilter).configure(conf).tap { |d| d.instance.start }
|
69
73
|
end
|
70
74
|
end
|
@@ -7,6 +7,10 @@ class JqFormatterTest < Test::Unit::TestCase
|
|
7
7
|
Fluent::Test.setup
|
8
8
|
end
|
9
9
|
|
10
|
+
teardown do
|
11
|
+
@driver.instance.shutdown if @driver
|
12
|
+
end
|
13
|
+
|
10
14
|
test "it should require jq parameter" do
|
11
15
|
assert_raise(Fluent::ConfigError) { create_driver '' }
|
12
16
|
end
|
@@ -48,6 +52,6 @@ class JqFormatterTest < Test::Unit::TestCase
|
|
48
52
|
private
|
49
53
|
|
50
54
|
def create_driver(conf)
|
51
|
-
Fluent::Test::Driver::Formatter.new(Fluent::Plugin::JqFormatter).configure(conf)
|
55
|
+
@driver = Fluent::Test::Driver::Formatter.new(Fluent::Plugin::JqFormatter).configure(conf).tap { |d| d.instance.start }
|
52
56
|
end
|
53
57
|
end
|
@@ -9,6 +9,10 @@ class JqParserTest < Test::Unit::TestCase
|
|
9
9
|
Fluent::Test.setup
|
10
10
|
end
|
11
11
|
|
12
|
+
teardown do
|
13
|
+
@driver.instance.shutdown if @driver
|
14
|
+
end
|
15
|
+
|
12
16
|
test "it should require jq" do
|
13
17
|
assert_raise(Fluent::ConfigError) { create_driver '' }
|
14
18
|
end
|
@@ -39,6 +43,6 @@ class JqParserTest < Test::Unit::TestCase
|
|
39
43
|
private
|
40
44
|
|
41
45
|
def create_driver(conf)
|
42
|
-
Fluent::Test::Driver::Parser.new(Fluent::Plugin::JqParser).configure(conf)
|
46
|
+
@driver = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JqParser).configure(conf).tap { |d| d.instance.start }
|
43
47
|
end
|
44
48
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-jq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zhimin (Gimi) Liang
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-04-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -86,20 +86,6 @@ dependencies:
|
|
86
86
|
- - "<"
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '2'
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
|
-
name: ruby-jq
|
91
|
-
requirement: !ruby/object:Gem::Requirement
|
92
|
-
requirements:
|
93
|
-
- - "~>"
|
94
|
-
- !ruby/object:Gem::Version
|
95
|
-
version: '0.1'
|
96
|
-
type: :runtime
|
97
|
-
prerelease: false
|
98
|
-
version_requirements: !ruby/object:Gem::Requirement
|
99
|
-
requirements:
|
100
|
-
- - "~>"
|
101
|
-
- !ruby/object:Gem::Version
|
102
|
-
version: '0.1'
|
103
89
|
- !ruby/object:Gem::Dependency
|
104
90
|
name: multi_json
|
105
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -135,6 +121,7 @@ files:
|
|
135
121
|
- fluent-plugin-jq.gemspec
|
136
122
|
- lib/fluent/plugin/filter_jq_transformer.rb
|
137
123
|
- lib/fluent/plugin/formatter_jq.rb
|
124
|
+
- lib/fluent/plugin/jq_mixin.rb
|
138
125
|
- lib/fluent/plugin/out_jq.rb
|
139
126
|
- lib/fluent/plugin/parser_jq.rb
|
140
127
|
- run_ci.sh
|