fluent-plugin-json-by-name 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +50 -0
- data/Makefile +43 -0
- data/README.md +67 -0
- data/Rakefile +11 -0
- data/build.sh +2 -0
- data/fluent_plugin_json_by_name.gemspec +12 -0
- data/lib/fluent/plugin/filter_json_by_name.rb +62 -0
- data/lib/release/fluent_plugin_json_by_name/version.rb +3 -0
- data/package.json +9 -0
- data/pkg/fluent-plugin-json-by-name-0.1.0.gem +0 -0
- data/publish.sh +3 -0
- data/test/helper.rb +12 -0
- data/test/plugin/test_filter_json_by_name.rb +92 -0
- metadata +115 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: de1b7254b02e90e313f58420fc68d907c3bf430f03f6cd4e5e9c9fb5817114d2
|
4
|
+
data.tar.gz: 231703f04dda952f44f30af034ba167a5939498dec830f0e0377b2a4e0501a3a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 359a8700afc9d93d1648faf8d88a24e88ac0a790725eaf0c6d1e07adab794566b4f1895a2ec9a9572ba9191a9e1ec8775a89dda1c34c73b2b82d940eaa419013
|
7
|
+
data.tar.gz: e57c8a3305e04187ce93dfb9345ff456d95f5365d8d0229ec4dd2b613b008d42ab4a4dc1241e1d0e5b6f1bc6d97f10db327cd9933f603459bb2a6eb845196b73
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fluent-plugin-json-by-name (0.1.0)
|
5
|
+
fluentd (~> 1.15, >= 1.15.3)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
concurrent-ruby (1.2.2)
|
11
|
+
cool.io (1.7.1)
|
12
|
+
fluentd (1.15.3)
|
13
|
+
bundler
|
14
|
+
cool.io (>= 1.4.5, < 2.0.0)
|
15
|
+
http_parser.rb (>= 0.5.1, < 0.9.0)
|
16
|
+
msgpack (>= 1.3.1, < 2.0.0)
|
17
|
+
serverengine (>= 2.3.0, < 3.0.0)
|
18
|
+
sigdump (~> 0.2.2)
|
19
|
+
strptime (>= 0.2.4, < 1.0.0)
|
20
|
+
tzinfo (>= 1.0, < 3.0)
|
21
|
+
tzinfo-data (~> 1.0)
|
22
|
+
webrick (>= 1.4.2, < 1.8.0)
|
23
|
+
yajl-ruby (~> 1.0)
|
24
|
+
http_parser.rb (0.8.0)
|
25
|
+
msgpack (1.6.1)
|
26
|
+
power_assert (2.0.2)
|
27
|
+
rake (13.0.6)
|
28
|
+
serverengine (2.3.2)
|
29
|
+
sigdump (~> 0.2.2)
|
30
|
+
sigdump (0.2.4)
|
31
|
+
strptime (0.2.5)
|
32
|
+
test-unit (3.5.5)
|
33
|
+
power_assert
|
34
|
+
tzinfo (2.0.6)
|
35
|
+
concurrent-ruby (~> 1.0)
|
36
|
+
tzinfo-data (1.2023.1)
|
37
|
+
tzinfo (>= 1.0.0)
|
38
|
+
webrick (1.7.0)
|
39
|
+
yajl-ruby (1.4.3)
|
40
|
+
|
41
|
+
PLATFORMS
|
42
|
+
x86_64-linux
|
43
|
+
|
44
|
+
DEPENDENCIES
|
45
|
+
fluent-plugin-json-by-name!
|
46
|
+
rake (~> 13.0, >= 13.0.6)
|
47
|
+
test-unit (~> 3.1, >= 3.1.0)
|
48
|
+
|
49
|
+
BUNDLED WITH
|
50
|
+
2.3.7
|
data/Makefile
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
.PHONY: deps
|
2
|
+
deps: deps-npm deps-ruby
|
3
|
+
|
4
|
+
.PHONY: deps-npm
|
5
|
+
deps-npm: node_modules
|
6
|
+
|
7
|
+
node_modules: package.json
|
8
|
+
npm install
|
9
|
+
|
10
|
+
.PHONY: deps-ruby
|
11
|
+
deps-ruby:
|
12
|
+
bundle install
|
13
|
+
|
14
|
+
.PHONY: lint
|
15
|
+
lint: lint-md
|
16
|
+
|
17
|
+
.PHONY: lint-fix
|
18
|
+
lint-fix: lint-md-fix
|
19
|
+
|
20
|
+
.PHONY: lint-md
|
21
|
+
lint-md: node_modules
|
22
|
+
npx remark .
|
23
|
+
|
24
|
+
.PHONY: lint-md-fix
|
25
|
+
lint-md-fix: node_modules
|
26
|
+
npx remark . -o
|
27
|
+
|
28
|
+
.PHONY: build
|
29
|
+
build: build-ruby
|
30
|
+
|
31
|
+
.PHONY: build-ruby
|
32
|
+
build-ruby:
|
33
|
+
rake build
|
34
|
+
|
35
|
+
.PHONY: test
|
36
|
+
test: test-ruby
|
37
|
+
|
38
|
+
.PHONY: test-ruby
|
39
|
+
test-ruby:
|
40
|
+
rake test
|
41
|
+
|
42
|
+
.PHONY: all
|
43
|
+
all: deps-ruby build
|
data/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
# fluent-plugin-json-by-name
|
2
|
+
|
3
|
+
## what does this plugin do
|
4
|
+
|
5
|
+
With this plugin you can add json fields as fluent field by the name of the
|
6
|
+
json field
|
7
|
+
|
8
|
+
## Configuration
|
9
|
+
|
10
|
+
For just indexing, provide the fields with value null in json:
|
11
|
+
|
12
|
+
```xml
|
13
|
+
<filter **>
|
14
|
+
@type json_by_name
|
15
|
+
fields_to_index {
|
16
|
+
"test1": null,
|
17
|
+
"test2": null,
|
18
|
+
"test3": null
|
19
|
+
}
|
20
|
+
</filter>
|
21
|
+
```
|
22
|
+
|
23
|
+
If you also want to rename the field, provide a new name as string instead of
|
24
|
+
null. All this also works for nested fields:
|
25
|
+
|
26
|
+
```xml
|
27
|
+
<filter **>
|
28
|
+
@type json_by_name
|
29
|
+
fields_to_index {
|
30
|
+
"test1": null,
|
31
|
+
"test2": "this_was_field_test2",
|
32
|
+
"nested": {
|
33
|
+
"test3": null,
|
34
|
+
"test4": "this_was_nested_field_test4"
|
35
|
+
}
|
36
|
+
}
|
37
|
+
</filter>
|
38
|
+
```
|
39
|
+
|
40
|
+
Assuming the above config, then the plugin would act as follows:
|
41
|
+
|
42
|
+
### Inputs
|
43
|
+
|
44
|
+
```json
|
45
|
+
{"message": "{\"test1\": true}"}
|
46
|
+
{"message": "{\"test2\": true}"}
|
47
|
+
{"message": "{\"nested\": {\"test3\": true}}"}
|
48
|
+
{"message": "{\"nested\": {\"test4\": true}}"}
|
49
|
+
```
|
50
|
+
|
51
|
+
### Outputs
|
52
|
+
|
53
|
+
```json
|
54
|
+
{"message": "{\"test1\": true}", "test1": true}
|
55
|
+
{"message": "{\"test2\": true}", "this_was_field_test2": true}
|
56
|
+
{"message": "{\"nested\": {\"test3\": true}}", "nested.test3": true}
|
57
|
+
{"message": "{\"nested\": {\"test4\": true}}", "this_was_nested_field_test4": true}
|
58
|
+
```
|
59
|
+
|
60
|
+
More examples can be found in `test/plugin/test_filter_json_by_name.rb`
|
61
|
+
|
62
|
+
## Testing
|
63
|
+
|
64
|
+
```sh
|
65
|
+
bundle install
|
66
|
+
rake
|
67
|
+
```
|
data/Rakefile
ADDED
data/build.sh
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require_relative 'lib/release/fluent_plugin_json_by_name/version'
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = 'fluent-plugin-json-by-name'
|
5
|
+
gem.version = FluentPluginJsonByName::VERSION
|
6
|
+
gem.summary = "Fluentd filter plugin to reshape json"
|
7
|
+
gem.authors = "platform+rubygems@riskident.com"
|
8
|
+
gem.files = `find *`.split("\n").uniq.sort.select{|f| !f.empty? }
|
9
|
+
gem.add_runtime_dependency 'fluentd', '~> 1.15', '>= 1.15.3'
|
10
|
+
gem.add_development_dependency 'rake', '~> 13.0', '>= 13.0.6'
|
11
|
+
gem.add_development_dependency "test-unit", '~> 3.1', '>= 3.1.0'
|
12
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'fluent/plugin/filter'
|
2
|
+
|
3
|
+
module Fluent::Plugin
|
4
|
+
|
5
|
+
class JsonByName < Filter
|
6
|
+
Fluent::Plugin.register_filter('json_by_name', self)
|
7
|
+
|
8
|
+
helpers :record_accessor
|
9
|
+
|
10
|
+
desc 'A hash/JSON object of fields to index and rename from log messages.'
|
11
|
+
config_param :fields_to_index, :hash, default: nil
|
12
|
+
|
13
|
+
def configure(conf)
|
14
|
+
super
|
15
|
+
validate_config(fields_to_index)
|
16
|
+
end
|
17
|
+
|
18
|
+
def combine_path(path, element)
|
19
|
+
return element if path.nil?
|
20
|
+
return "#{path}.#{element}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def validate_config(fields_to_index_remaining, path=nil)
|
24
|
+
if fields_to_index_remaining.class == Hash then
|
25
|
+
fields_to_index_remaining.keys.each do |key|
|
26
|
+
validate_config(fields_to_index_remaining[key], combine_path(path, key))
|
27
|
+
end
|
28
|
+
return
|
29
|
+
elsif fields_to_index_remaining.nil? or fields_to_index_remaining.class == String then
|
30
|
+
return
|
31
|
+
end
|
32
|
+
raise Fluent::ConfigError.new("value at \"#{path}\" in config is not either object, string, or null")
|
33
|
+
end
|
34
|
+
|
35
|
+
def reshape(record, fields_to_index_remaining, data, path=nil)
|
36
|
+
return if data == nil
|
37
|
+
if fields_to_index_remaining.class == Hash and data.class == Hash then
|
38
|
+
common_keys = Set.new(fields_to_index_remaining.keys) & Set.new(data.keys)
|
39
|
+
common_keys.each do |key|
|
40
|
+
reshape(record, fields_to_index_remaining[key], data[key], combine_path(path, key))
|
41
|
+
end
|
42
|
+
elsif fields_to_index_remaining.nil? then
|
43
|
+
record[path] = data
|
44
|
+
elsif fields_to_index_remaining.class == String then
|
45
|
+
record[fields_to_index_remaining] = data
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def filter(tag, time, record)
|
50
|
+
return record if record["message"] == nil
|
51
|
+
begin
|
52
|
+
record_as_json = JSON.parse(record["message"])
|
53
|
+
rescue JSON::ParserError
|
54
|
+
return record
|
55
|
+
end
|
56
|
+
|
57
|
+
reshape(record, fields_to_index, record_as_json)
|
58
|
+
|
59
|
+
return record
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/package.json
ADDED
Binary file
|
data/publish.sh
ADDED
data/test/helper.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'test/unit'
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
5
|
+
$LOAD_PATH.unshift(__dir__)
|
6
|
+
require 'fluent/test'
|
7
|
+
require 'fluent/test/driver/filter'
|
8
|
+
require 'fluent/test/helpers'
|
9
|
+
|
10
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
11
|
+
|
12
|
+
require 'fluent/plugin/filter_json_by_name'
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class JsonByNameTest < Test::Unit::TestCase
|
4
|
+
include Fluent::Test::Helpers
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
CONFIG = %[
|
10
|
+
fields_to_index {
|
11
|
+
"test1": null,
|
12
|
+
"test2": "this_was_field_test2",
|
13
|
+
"nested": {
|
14
|
+
"test3": null,
|
15
|
+
"test4": "this_was_field_test4"
|
16
|
+
},
|
17
|
+
"nonnested.test5": null,
|
18
|
+
"nonnested.test6": "this_was_field_test6"
|
19
|
+
}
|
20
|
+
]
|
21
|
+
|
22
|
+
def create_driver(config)
|
23
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::JsonByName).configure(config)
|
24
|
+
end
|
25
|
+
|
26
|
+
def run_plugin(message, config = CONFIG)
|
27
|
+
d = create_driver(config)
|
28
|
+
d.run(default_tag: "test") do
|
29
|
+
d.feed(Fluent::Engine.now, {"message" => message})
|
30
|
+
end
|
31
|
+
return d.filtered_records
|
32
|
+
end
|
33
|
+
|
34
|
+
def assert_message(original_message, result, added_fields)
|
35
|
+
message = {"message" => original_message}
|
36
|
+
full_message = message.merge(added_fields)
|
37
|
+
assert_equal([full_message], result)
|
38
|
+
end
|
39
|
+
|
40
|
+
def run_and_assert(message, added_fields, config = CONFIG)
|
41
|
+
result = run_plugin(message, config)
|
42
|
+
assert_message(message, result, added_fields)
|
43
|
+
end
|
44
|
+
|
45
|
+
test 'valid config validates' do
|
46
|
+
assert_nothing_raised(RuntimeError) {
|
47
|
+
create_driver(CONFIG)
|
48
|
+
}
|
49
|
+
end
|
50
|
+
|
51
|
+
test 'invalid config fails validation' do
|
52
|
+
assert_raise(Fluent::ConfigError.new("value at \"foo.invalid_config\" in config is not either object, string, or null")) {
|
53
|
+
create_driver(<<-CONFIG)
|
54
|
+
fields_to_index {
|
55
|
+
"foo": {
|
56
|
+
"rename_given_name": "renamename",
|
57
|
+
"invalid_config" : true,
|
58
|
+
"rename_same_name" : null
|
59
|
+
}
|
60
|
+
}
|
61
|
+
CONFIG
|
62
|
+
}
|
63
|
+
end
|
64
|
+
|
65
|
+
test 'invalid json is left as is' do
|
66
|
+
run_and_assert("this_is_not_json", {})
|
67
|
+
end
|
68
|
+
|
69
|
+
test 'indexing field' do
|
70
|
+
run_and_assert("{\"test1\":true}", {"test1" => true})
|
71
|
+
end
|
72
|
+
|
73
|
+
test 'indexing and renaming field' do
|
74
|
+
run_and_assert("{\"test2\":true}", {"this_was_field_test2" => true})
|
75
|
+
end
|
76
|
+
|
77
|
+
test 'indexing nested field' do
|
78
|
+
run_and_assert("{\"nested\":{\"test3\":true}}", {"nested.test3" => true})
|
79
|
+
end
|
80
|
+
|
81
|
+
test 'indexing and renaming nested field' do
|
82
|
+
run_and_assert("{\"nested\":{\"test4\":true}}", {"this_was_field_test4" => true})
|
83
|
+
end
|
84
|
+
|
85
|
+
test 'indexing field with "." in name' do
|
86
|
+
run_and_assert("{\"nonnested.test5\":true}", {"nonnested.test5" => true})
|
87
|
+
end
|
88
|
+
|
89
|
+
test 'indexing and renaming field with "." in name' do
|
90
|
+
run_and_assert("{\"nonnested.test6\":true}", {"this_was_field_test6" => true})
|
91
|
+
end
|
92
|
+
end
|
metadata
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-json-by-name
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- platform+rubygems@riskident.com
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-03-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.15'
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 1.15.3
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - "~>"
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.15'
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.15.3
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: rake
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '13.0'
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 13.0.6
|
43
|
+
type: :development
|
44
|
+
prerelease: false
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '13.0'
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 13.0.6
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: test-unit
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - "~>"
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '3.1'
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: 3.1.0
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '3.1'
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: 3.1.0
|
73
|
+
description:
|
74
|
+
email:
|
75
|
+
executables: []
|
76
|
+
extensions: []
|
77
|
+
extra_rdoc_files: []
|
78
|
+
files:
|
79
|
+
- Gemfile
|
80
|
+
- Gemfile.lock
|
81
|
+
- Makefile
|
82
|
+
- README.md
|
83
|
+
- Rakefile
|
84
|
+
- build.sh
|
85
|
+
- fluent_plugin_json_by_name.gemspec
|
86
|
+
- lib/fluent/plugin/filter_json_by_name.rb
|
87
|
+
- lib/release/fluent_plugin_json_by_name/version.rb
|
88
|
+
- package.json
|
89
|
+
- pkg/fluent-plugin-json-by-name-0.1.0.gem
|
90
|
+
- publish.sh
|
91
|
+
- test/helper.rb
|
92
|
+
- test/plugin/test_filter_json_by_name.rb
|
93
|
+
homepage:
|
94
|
+
licenses: []
|
95
|
+
metadata: {}
|
96
|
+
post_install_message:
|
97
|
+
rdoc_options: []
|
98
|
+
require_paths:
|
99
|
+
- lib
|
100
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
requirements: []
|
111
|
+
rubygems_version: 3.3.26
|
112
|
+
signing_key:
|
113
|
+
specification_version: 4
|
114
|
+
summary: Fluentd filter plugin to reshape json
|
115
|
+
test_files: []
|