fluent-plugin-to-flat-json-parser 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: cf6d1209e2e16cf71dc8c6f4b3a15527908488e7a81ee21a85e73749d29a56c2
4
+ data.tar.gz: 1f0389520bb0e686be1d4bd5b0d4f32c4fe4176772e725b2d83669cee73f630e
5
+ SHA512:
6
+ metadata.gz: 6b15055aa2ba084d26b6ebbb7000a3a87d842e27b34d7b288e2e4d175806425593f09a6213feffe76298c172a05ab61c94195673f4a156cc33aeddb0f567cdf2
7
+ data.tar.gz: 9dbf90a465d111bf09aeddb7e8551ff60b49ec96c0a37460d7783c190db050be4007abb6f1c3b43b38d44dc7385aa6b0b37b54500293c33f293b904da97c4fd7
@@ -0,0 +1,30 @@
1
+ /*.gem
2
+ ~*
3
+ #*
4
+ *~
5
+ Gemfile.lock
6
+ .rbenv-version
7
+ vendor
8
+ doc/*
9
+ tmp/*
10
+ coverage
11
+ .yardoc
12
+ pkg/
13
+ .ruby-version
14
+ *.gem
15
+ *.rbc
16
+ .config
17
+ coverage
18
+ InstalledFiles
19
+ lib/bundler/man
20
+ pkg
21
+ rdoc
22
+ spec/reports
23
+ test/tmp
24
+ test/version_tmp
25
+ tmp
26
+
27
+ # YARD artifacts
28
+ .yardoc
29
+ _yardoc
30
+ doc/
@@ -0,0 +1,13 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 2.3.0
5
+ - 2.4.0
6
+ - 2.5.0
7
+ - 2.6.0
8
+
9
+ before_install:
10
+ - gem update bundler
11
+
12
+ script:
13
+ - bundle exec rake test
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2014 studio3104
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,44 @@
1
+ # fluent-plugin-to-flat-json-parser
2
+
3
+ fluentd parser plugin to create flattened JSON from nested JSON objects
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'fluent-plugin-to-flat-json-parser'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Readying for a package build:
16
+
17
+ $ bundle install
18
+ $ bundle exec rake build
19
+
20
+ ## Configuration
21
+
22
+ - configuration
23
+
24
+ ```
25
+ <source>
26
+ type tail
27
+ path /tmp/fluentd.json
28
+ pos_file /tmp/fluentd.log.pos
29
+ tag logs
30
+ format to_json_flat
31
+ time_key timestamp
32
+ separator .
33
+ </source>
34
+ ```
35
+
36
+ #### Parameter
37
+
38
+ ###### time_key
39
+ - Default is `time`
40
+ - Field to use for time resolution
41
+
42
+ ###### separator
43
+ - Default is '.'
44
+ - Used as the value to split the keys
@@ -0,0 +1,11 @@
1
+ # encoding: utf-8
2
+ require "bundler/gem_tasks"
3
+ require 'rake/testtask'
4
+
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,22 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = 'fluent-plugin-to-flat-json-parser'
6
+ gem.version = '0.0.1'
7
+ gem.authors = ['Al-waleed Shihadeh']
8
+ gem.email = 'wshihadh@gmail.com'
9
+ gem.homepage = 'https://github.com/shihadeh/fluent-plugin-to-flat-json-parser'
10
+ gem.description = 'fluentd parser plugin to flatten nested json objects'
11
+ gem.summary = gem.description
12
+ gem.licenses = ['MIT']
13
+
14
+ gem.files = `git ls-files`.split("\n")
15
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ gem.require_paths = ['lib']
18
+
19
+ gem.add_runtime_dependency 'fluentd', '~> 1.2.2'
20
+ gem.add_development_dependency 'test-unit'
21
+ gem.add_development_dependency 'rake'
22
+ end
@@ -0,0 +1,40 @@
1
+ require 'fluent/plugin/parser'
2
+
3
+ module Fluent::Plugin
4
+ class ToFlatJsonParser < JSONParser
5
+ Fluent::Plugin.register_parser("to_flat_json", self)
6
+
7
+ config_param :separator, :string, default: '.'
8
+
9
+ def parse(text)
10
+ r = @load_proc.call(text)
11
+ time, record = convert_values(parse_time(r), r)
12
+ record = record_flatten(record)
13
+ yield time, record
14
+ rescue @error_class, EncodingError
15
+ yield nil, nil
16
+ end
17
+
18
+ def record_flatten(record, parent = nil)
19
+ flattend_record = {}
20
+ record.each_with_index do |obj, i|
21
+ if obj.is_a?(Array)
22
+ k, v = obj
23
+ else
24
+ k, v = i, obj
25
+ end
26
+
27
+ key = parent ? "#{parent}#{@separator}#{k}" : k
28
+ key = key.gsub('.', @separator)
29
+
30
+ if v.is_a? Enumerable
31
+ flattend_record.merge!(record_flatten(v, key))
32
+ else
33
+ flattend_record[key] = v
34
+ end
35
+ end
36
+
37
+ flattend_record
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,8 @@
1
+ require 'test/unit'
2
+ require 'fluent/log'
3
+ require 'fluent/test'
4
+
5
+ unless defined?(Test::Unit::AssertionFailedError)
6
+ class Test::Unit::AssertionFailedError < StandardError
7
+ end
8
+ end
@@ -0,0 +1,78 @@
1
+ require 'helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser_to_flat_json'
4
+
5
+ class ParserToFlatJson < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::ToFlatJsonParser)
9
+ end
10
+
11
+ data('oj' => 'oj', 'yajl' => 'yajl')
12
+ def test_parse(data)
13
+ @parser.configure('json_parser' => data)
14
+ @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
15
+ assert_equal(Fluent::EventTime.parse('2013-02-28 12:00:00 +0900').to_i, time)
16
+ assert_equal({
17
+ 'host' => '192.168.0.1',
18
+ 'size' => 777,
19
+ 'method' => 'PUT',
20
+ }, record)
21
+ }
22
+ end
23
+
24
+ data('oj' => 'oj', 'yajl' => 'yajl')
25
+ def test_parse_timestamp(data)
26
+ @parser.configure('json_parser' => data, 'time_key' => 'timestamp')
27
+ @parser.instance.parse('{"timestamp":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
28
+ assert_equal(Fluent::EventTime.parse('2013-02-28 12:00:00 +0900').to_i, time)
29
+ assert_equal({
30
+ 'host' => '192.168.0.1',
31
+ 'size' => 777,
32
+ 'method' => 'PUT',
33
+ }, record)
34
+ }
35
+ end
36
+
37
+
38
+ data('oj' => 'oj', 'yajl' => 'yajl')
39
+ def test_nasted_parse(data)
40
+ @parser.configure('json_parser' => data)
41
+ @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","obj":{"size":777,"method":"PUT"}') { |time, record|
42
+ assert_equal(Fluent::EventTime.parse('2013-02-28 12:00:00 +0900').to_i, time)
43
+ assert_equal({
44
+ 'host' => '192.168.0.1',
45
+ 'obj.size' => 777,
46
+ 'obj.method' => 'PUT',
47
+ }, record)
48
+ }
49
+ end
50
+
51
+ data('oj' => 'oj', 'yajl' => 'yajl')
52
+ def test_nasted_separator_parse(data)
53
+ @parser.configure('json_parser' => data, 'separator' => '_')
54
+ @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","obj":{"size":777,"method":"PUT"}') { |time, record|
55
+ assert_equal(Fluent::EventTime.parse('2013-02-28 12:00:00 +0900').to_i, time)
56
+ assert_equal({
57
+ 'host' => '192.168.0.1',
58
+ 'obj_size' => 777,
59
+ 'obj_method' => 'PUT',
60
+ }, record)
61
+ }
62
+ end
63
+
64
+ data('oj' => 'oj', 'yajl' => 'yajl')
65
+ def test_nasted_array_with_separator_parse(data)
66
+ @parser.configure('json_parser' => data, 'separator' => '_')
67
+ @parser.instance.parse('{"time":1362020400,"host":"192.168.0.1","mlist":["x","y"],"obj":{"size":777,"method":"PUT"}') { |time, record|
68
+ assert_equal(Fluent::EventTime.parse('2013-02-28 12:00:00 +0900').to_i, time)
69
+ assert_equal({
70
+ 'host' => '192.168.0.1',
71
+ 'obj_size' => 777,
72
+ 'obj_method' => 'PUT',
73
+ 'mlist_0' => 'x',
74
+ 'mlist_1' => 'y',
75
+ }, record)
76
+ }
77
+ end
78
+ end
metadata ADDED
@@ -0,0 +1,97 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-to-flat-json-parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Al-waleed Shihadeh
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-07-02 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 1.2.2
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 1.2.2
27
+ - !ruby/object:Gem::Dependency
28
+ name: test-unit
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: fluentd parser plugin to flatten nested json objects
56
+ email: wshihadh@gmail.com
57
+ executables: []
58
+ extensions: []
59
+ extra_rdoc_files: []
60
+ files:
61
+ - ".gitignore"
62
+ - ".travis.yml"
63
+ - Gemfile
64
+ - LICENSE
65
+ - README.md
66
+ - Rakefile
67
+ - fluent-plugin-to-flat-json-parser.gemspec
68
+ - lib/fluent/plugin/parser_to_flat_json.rb
69
+ - test/helper.rb
70
+ - test/plugin/test_parser-to-flat-json.rb
71
+ homepage: https://github.com/shihadeh/fluent-plugin-to-flat-json-parser
72
+ licenses:
73
+ - MIT
74
+ metadata: {}
75
+ post_install_message:
76
+ rdoc_options: []
77
+ require_paths:
78
+ - lib
79
+ required_ruby_version: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - ">="
82
+ - !ruby/object:Gem::Version
83
+ version: '0'
84
+ required_rubygems_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ requirements: []
90
+ rubyforge_project:
91
+ rubygems_version: 2.7.8
92
+ signing_key:
93
+ specification_version: 4
94
+ summary: fluentd parser plugin to flatten nested json objects
95
+ test_files:
96
+ - test/helper.rb
97
+ - test/plugin/test_parser-to-flat-json.rb