fluent-plugin-serialize-nested-json 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/README.md +48 -0
- data/Rakefile +10 -0
- data/fluent-plugin-serialize-nested-json.gemspec +25 -0
- data/lib/fluent/plugin/parser_serialize_nested_json.rb +45 -0
- data/test/helper.rb +128 -0
- data/test/test_parser.rb +67 -0
- metadata +144 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9f869e0a7533c134e0597890a3de49ed7ffc8844
|
4
|
+
data.tar.gz: ab55ad2cdc152698eaebe663c2610b167465f424
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 61587db062114ed826461a9a836d8f334c9a4b93912176dc6546fb4064f0e8f20cd67ad06be2929784c7375ba35a11ced059d22fd314ea7e1788664b9390b6ea
|
7
|
+
data.tar.gz: 57e5e02f236cdc1272fe8081c56f5ebbe18621c5f26ba0d4bade897e4f911f762033f16023b7469426cc6c27a347af89eb268ffae087d7ed04c5d83184500a46
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
# fluent-plugin-json-in-json [![Gem Version](https://badge.fury.io/rb/fluent-plugin-json-in-json-2.svg)](https://badge.fury.io/rb/fluent-plugin-json-in-json-2)
|
2
|
+
|
3
|
+
This is a fork repo of [arcivanov/fluent-plugin-json-in-json](https://github.com/arcivanov/fluent-plugin-json-in-json) which itself fork of [gmr/fluent-plugin-json-in-json](https://github.com/gmr/fluent-plugin-json-in-json) which is not supported now.
|
4
|
+
|
5
|
+
The purpose of this repo to publish actual version of `fluent-plugin-json-in-json` plugin.
|
6
|
+
|
7
|
+
## Requirements
|
8
|
+
|
9
|
+
This fluentd parser plugin parses JSON log lines with nested JSON strings. For
|
10
|
+
example, given a docker log of ``{"log": "{\"foo\": \"bar\"}"}``, the log record
|
11
|
+
will be parsed into ``{:log => { :foo => "bar" }}``.
|
12
|
+
|
13
|
+
## Installation
|
14
|
+
|
15
|
+
Add this line to your application's Gemfile:
|
16
|
+
|
17
|
+
gem 'fluent-plugin-json-in-json-2'
|
18
|
+
|
19
|
+
And then execute:
|
20
|
+
|
21
|
+
$ bundle
|
22
|
+
|
23
|
+
Or install it yourself as:
|
24
|
+
|
25
|
+
$ gem install fluent-plugin-json-in-json-2
|
26
|
+
|
27
|
+
|
28
|
+
## Usage
|
29
|
+
|
30
|
+
```
|
31
|
+
<source>
|
32
|
+
type tail
|
33
|
+
path /var/lib/docker/containers/*/*-json.log
|
34
|
+
pos_file /var/log/fluentd-docker.pos
|
35
|
+
time_format %Y-%m-%dT%H:%M:%S
|
36
|
+
tag docker.*
|
37
|
+
format json_in_json
|
38
|
+
read_from_head true
|
39
|
+
</source>
|
40
|
+
```
|
41
|
+
|
42
|
+
## Contributing
|
43
|
+
|
44
|
+
1. Fork it
|
45
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
46
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
47
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
48
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
Gem::Specification.new do |spec|
|
3
|
+
spec.name = "fluent-plugin-serialize-nested-json"
|
4
|
+
spec.version = "0.0.1"
|
5
|
+
spec.authors = ["Yagnesh Mistry"]
|
6
|
+
spec.email = ["ysh@live.in"]
|
7
|
+
spec.description = %q{Parser plugin that serializes nested JSON attributes}
|
8
|
+
spec.summary = %q{Parser plugin that serializes nested JSON attributes}
|
9
|
+
spec.homepage = "https://github.com/ysh7/fluent-plugin-serialize-nested-json"
|
10
|
+
spec.license = "BSD"
|
11
|
+
|
12
|
+
spec.files = `git ls-files`.split($/)
|
13
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
14
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
15
|
+
spec.require_paths = ["lib"]
|
16
|
+
|
17
|
+
spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
18
|
+
spec.add_runtime_dependency 'yajl-ruby', '~> 1.0'
|
19
|
+
|
20
|
+
spec.add_development_dependency 'rake', '~> 12.3'
|
21
|
+
spec.add_development_dependency 'bundler', '~> 1.16'
|
22
|
+
|
23
|
+
spec.add_development_dependency 'test-unit', ['~> 3.2']
|
24
|
+
spec.add_development_dependency 'test-unit-rr', ['~> 1.0']
|
25
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'fluent/parser'
|
2
|
+
require 'yajl'
|
3
|
+
|
4
|
+
module Fluent
|
5
|
+
module Plugin
|
6
|
+
class SerializeJSONParser < Parser
|
7
|
+
Plugin.register_parser('serialize_nested_json', self)
|
8
|
+
|
9
|
+
config_set_default :time_key, 'time'
|
10
|
+
config_set_default :time_type, :float
|
11
|
+
|
12
|
+
def configure(conf)
|
13
|
+
if conf.has_key?('time_format')
|
14
|
+
conf['time_type'] ||= 'string'
|
15
|
+
end
|
16
|
+
|
17
|
+
super
|
18
|
+
end
|
19
|
+
|
20
|
+
def parse(text)
|
21
|
+
record = Yajl.load(text)
|
22
|
+
|
23
|
+
values = Hash.new
|
24
|
+
|
25
|
+
record.each do |k, v|
|
26
|
+
if v.is_a?(Hash) || v.is_a?(Array)
|
27
|
+
begin
|
28
|
+
values[k] = Yajl::Encoder.encode(v)
|
29
|
+
record.delete k
|
30
|
+
rescue Exception => e
|
31
|
+
# continue
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
record.merge!(values)
|
36
|
+
|
37
|
+
time, record = convert_values(parse_time(record), record)
|
38
|
+
|
39
|
+
yield time, record
|
40
|
+
rescue Yajl::ParseError
|
41
|
+
yield nil, nil
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
# Some tests use Hash instead of Element for configure.
|
2
|
+
# We should rewrite these tests in the future and remove this ad-hoc code
|
3
|
+
class Hash
|
4
|
+
def corresponding_proxies
|
5
|
+
@corresponding_proxies ||= []
|
6
|
+
end
|
7
|
+
|
8
|
+
def to_masked_element
|
9
|
+
self
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
require 'rr'
|
14
|
+
require 'test/unit'
|
15
|
+
require 'test/unit/rr'
|
16
|
+
require 'fileutils'
|
17
|
+
require 'fluent/config/element'
|
18
|
+
require 'fluent/log'
|
19
|
+
require 'fluent/test'
|
20
|
+
require 'fluent/test/helpers'
|
21
|
+
require 'fluent/plugin/base'
|
22
|
+
require 'fluent/plugin_id'
|
23
|
+
require 'fluent/plugin_helper'
|
24
|
+
require 'fluent/msgpack_factory'
|
25
|
+
require 'fluent/time'
|
26
|
+
require 'serverengine'
|
27
|
+
|
28
|
+
module Fluent
|
29
|
+
module Plugin
|
30
|
+
class TestBase < Base
|
31
|
+
# a base plugin class, but not input nor output
|
32
|
+
# mainly for helpers and owned plugins
|
33
|
+
include PluginId
|
34
|
+
include PluginLoggerMixin
|
35
|
+
include PluginHelper::Mixin
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
unless defined?(Test::Unit::AssertionFailedError)
|
41
|
+
class Test::Unit::AssertionFailedError < StandardError
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
include Fluent::Test::Helpers
|
46
|
+
|
47
|
+
def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
|
48
|
+
case protocol
|
49
|
+
when :tcp
|
50
|
+
unused_port_tcp(num)
|
51
|
+
when :udp
|
52
|
+
unused_port_udp(num, bind: bind)
|
53
|
+
else
|
54
|
+
raise ArgumentError, "unknown protocol: #{protocol}"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def unused_port_tcp(num = 1)
|
59
|
+
ports = []
|
60
|
+
sockets = []
|
61
|
+
num.times do
|
62
|
+
s = TCPServer.open(0)
|
63
|
+
sockets << s
|
64
|
+
ports << s.addr[1]
|
65
|
+
end
|
66
|
+
sockets.each{|s| s.close }
|
67
|
+
if num == 1
|
68
|
+
return ports.first
|
69
|
+
else
|
70
|
+
return *ports
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
PORT_RANGE_AVAILABLE = (1024...65535)
|
75
|
+
|
76
|
+
def unused_port_udp(num = 1, bind: "0.0.0.0")
|
77
|
+
family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
|
78
|
+
ports = []
|
79
|
+
sockets = []
|
80
|
+
while ports.size < num
|
81
|
+
port = rand(PORT_RANGE_AVAILABLE)
|
82
|
+
u = UDPSocket.new(family)
|
83
|
+
if (u.bind(bind, port) rescue nil)
|
84
|
+
ports << port
|
85
|
+
sockets << u
|
86
|
+
else
|
87
|
+
u.close
|
88
|
+
end
|
89
|
+
end
|
90
|
+
sockets.each{|s| s.close }
|
91
|
+
if num == 1
|
92
|
+
return ports.first
|
93
|
+
else
|
94
|
+
return *ports
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def waiting(seconds, logs: nil, plugin: nil)
|
99
|
+
begin
|
100
|
+
Timeout.timeout(seconds) do
|
101
|
+
yield
|
102
|
+
end
|
103
|
+
rescue Timeout::Error
|
104
|
+
if logs
|
105
|
+
STDERR.print(*logs)
|
106
|
+
elsif plugin
|
107
|
+
STDERR.print(*plugin.log.out.logs)
|
108
|
+
end
|
109
|
+
raise
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def ipv6_enabled?
|
114
|
+
require 'socket'
|
115
|
+
|
116
|
+
begin
|
117
|
+
TCPServer.open("::1", 0)
|
118
|
+
true
|
119
|
+
rescue
|
120
|
+
false
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
dl_opts = {}
|
125
|
+
dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
|
126
|
+
logdev = Fluent::Test::DummyLogDevice.new
|
127
|
+
logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
|
128
|
+
$log ||= Fluent::Log.new(logger)
|
data/test/test_parser.rb
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
require_relative 'helper'
|
2
|
+
require 'fluent/test/driver/parser'
|
3
|
+
require 'fluent/plugin/parser_json_in_json'
|
4
|
+
require 'yajl'
|
5
|
+
|
6
|
+
class JsonInJsonParserTest < ::Test::Unit::TestCase
|
7
|
+
def setup
|
8
|
+
Fluent::Test.setup
|
9
|
+
@parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONInJSONParser)
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_parse_float_time()
|
13
|
+
@parser.configure({})
|
14
|
+
@parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT","log_json":" {\\"field1\\":\\"field1 value\\",\\"field2\\":40}"}') { |time, record|
|
15
|
+
assert_equal(event_time('2013-02-28 12:00:00 +0900').to_f, time.to_f)
|
16
|
+
assert_equal({
|
17
|
+
'host' => '192.168.0.1',
|
18
|
+
'size' => 777,
|
19
|
+
'method' => 'PUT',
|
20
|
+
'field1' => 'field1 value',
|
21
|
+
'field2' => 40
|
22
|
+
}, record)
|
23
|
+
}
|
24
|
+
@parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT","log_json":"{\\"field1\\":"}') { |time, record|
|
25
|
+
assert_equal(event_time('2013-02-28 12:00:00 +0900').to_f, time.to_f)
|
26
|
+
assert_equal({
|
27
|
+
'host' => '192.168.0.1',
|
28
|
+
'size' => 777,
|
29
|
+
'method' => 'PUT',
|
30
|
+
'log_json' => '{"field1":'
|
31
|
+
}, record)
|
32
|
+
}
|
33
|
+
@parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT","log_json":"[\\"val1\\",\\"val2\\"]"') { |time, record|
|
34
|
+
assert_equal(event_time('2013-02-28 12:00:00 +0900').to_f, time.to_f)
|
35
|
+
assert_equal({
|
36
|
+
'host' => '192.168.0.1',
|
37
|
+
'size' => 777,
|
38
|
+
'method' => 'PUT',
|
39
|
+
'log_json' => ['val1', 'val2']
|
40
|
+
}, record)
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_parse_string_time()
|
45
|
+
@parser.configure('time_format' => '%Y-%m-%dT%H:%M:%S.%NZ', 'keep_time_key' => 'true')
|
46
|
+
@parser.instance.parse('{"log":"2018-06-26 13:20:44.075 INFO --- [pool-8-thread-3] outgoing","stream":"stdout","time":"2018-06-26T13:20:44.076022960Z"}') { |time, record|
|
47
|
+
assert_equal(event_time('2018-06-26 13:20:44.076022960').to_f, time.to_f)
|
48
|
+
assert_equal({
|
49
|
+
'log'=>'2018-06-26 13:20:44.075 INFO --- [pool-8-thread-3] outgoing',
|
50
|
+
'stream'=>'stdout',
|
51
|
+
'time'=>'2018-06-26T13:20:44.076022960Z'
|
52
|
+
}, record)
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_yajl_load()
|
57
|
+
@parser.configure({})
|
58
|
+
@parser.instance.parse('{ "log": " [ msg ] messoge [ k ]", "stream": "stdout"}') { |time, record|
|
59
|
+
assert_equal({
|
60
|
+
'log'=> ' [ msg ] messoge [ k ]',
|
61
|
+
'stream'=>'stdout',
|
62
|
+
}, record)
|
63
|
+
}
|
64
|
+
# Yajl.load('["kek":1}')
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
metadata
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-serialize-nested-json
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yagnesh Mistry
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-09-25 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.14.0
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '2'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.14.0
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: yajl-ruby
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '1.0'
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '1.0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rake
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '12.3'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '12.3'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: bundler
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - "~>"
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '1.16'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '1.16'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: test-unit
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - "~>"
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '3.2'
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '3.2'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
name: test-unit-rr
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - "~>"
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '1.0'
|
96
|
+
type: :development
|
97
|
+
prerelease: false
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - "~>"
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '1.0'
|
103
|
+
description: Parser plugin that serializes nested JSON attributes
|
104
|
+
email:
|
105
|
+
- ysh@live.in
|
106
|
+
executables: []
|
107
|
+
extensions: []
|
108
|
+
extra_rdoc_files: []
|
109
|
+
files:
|
110
|
+
- ".gitignore"
|
111
|
+
- Gemfile
|
112
|
+
- README.md
|
113
|
+
- Rakefile
|
114
|
+
- fluent-plugin-serialize-nested-json.gemspec
|
115
|
+
- lib/fluent/plugin/parser_serialize_nested_json.rb
|
116
|
+
- test/helper.rb
|
117
|
+
- test/test_parser.rb
|
118
|
+
homepage: https://github.com/ysh7/fluent-plugin-serialize-nested-json
|
119
|
+
licenses:
|
120
|
+
- BSD
|
121
|
+
metadata: {}
|
122
|
+
post_install_message:
|
123
|
+
rdoc_options: []
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
requirements: []
|
137
|
+
rubyforge_project:
|
138
|
+
rubygems_version: 2.5.2.1
|
139
|
+
signing_key:
|
140
|
+
specification_version: 4
|
141
|
+
summary: Parser plugin that serializes nested JSON attributes
|
142
|
+
test_files:
|
143
|
+
- test/helper.rb
|
144
|
+
- test/test_parser.rb
|