fluent-plugin-field-flatten-json 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +34 -0
- data/README.md +20 -0
- data/Rakefile +6 -0
- data/VERSION +1 -0
- data/fluent-plugin-field-multiregex.gemspec +24 -0
- data/lib/fluent/plugin/out_field_flatten_json.rb +66 -0
- metadata +92 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b08b1dc98e401ecd654c444f2abd4ef5d82258e2
|
4
|
+
data.tar.gz: 8cba69a2ed39dfbf591868cca3d0b4c50b25be94
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7eef2a29058f5afa3ce5eb820ce74a8f62e1bed08d11f62769032beb81a9066d81fd24a8e12a845572f8ee15a44d5750c76af1f1ac56a94541135aaf0f1a4e5f
|
7
|
+
data.tar.gz: 38155e5f519b13fdf040740b059a7b6bc671ef8ab82bad7650cd23f4a5e443d3f5fb94b6533a264669f03d92ffae382442f5e5abbb0141001ab08c3ffa62847e
|
data/.gitignore
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/test/tmp/
|
9
|
+
/test/version_tmp/
|
10
|
+
/tmp/
|
11
|
+
|
12
|
+
## Specific to RubyMotion:
|
13
|
+
.dat*
|
14
|
+
.repl_history
|
15
|
+
build/
|
16
|
+
|
17
|
+
## Documentation cache and generated files:
|
18
|
+
/.yardoc/
|
19
|
+
/_yardoc/
|
20
|
+
/doc/
|
21
|
+
/rdoc/
|
22
|
+
|
23
|
+
## Environment normalisation:
|
24
|
+
/.bundle/
|
25
|
+
/lib/bundler/man/
|
26
|
+
|
27
|
+
# for a library or gem, you might want to ignore these files since the code is
|
28
|
+
# intended to run in multiple environments; otherwise, check them in:
|
29
|
+
# Gemfile.lock
|
30
|
+
# .ruby-version
|
31
|
+
# .ruby-gemset
|
32
|
+
|
33
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
34
|
+
.rvmrc
|
data/README.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
fluent-plugin-field-flatten-json
|
2
|
+
===========================
|
3
|
+
|
4
|
+
Fluent output plugin for reforming a record using multiple named capture regular expressions
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
Use RubyGems:
|
9
|
+
|
10
|
+
td-agent-gem install fluent-plugin-field-flatten-json
|
11
|
+
|
12
|
+
## Configuration
|
13
|
+
|
14
|
+
<match pattern>
|
15
|
+
type field_flatten_json
|
16
|
+
flatten_key message
|
17
|
+
remove_tag_prefix raw
|
18
|
+
add_tag_prefix parsed
|
19
|
+
</match>
|
20
|
+
|
data/Rakefile
ADDED
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.1
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-field-flatten-json"
|
6
|
+
gem.description = "Fluent output plugin for flattening a json field"
|
7
|
+
gem.homepage = "https://github.com/CiscoZeus/fluent-plugin-field-flatten-json"
|
8
|
+
gem.summary = gem.description
|
9
|
+
gem.version = File.read("VERSION").strip
|
10
|
+
gem.authors = ["Manoj Sharma"]
|
11
|
+
gem.email = ["vigyanik@gmail.com"]
|
12
|
+
gem.has_rdoc = false
|
13
|
+
gem.license = 'MIT'
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split("\n")
|
16
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
gem.require_paths = ['lib']
|
19
|
+
|
20
|
+
gem.add_dependency "fluentd"
|
21
|
+
gem.add_dependency "json"
|
22
|
+
gem.add_development_dependency "rake"
|
23
|
+
end
|
24
|
+
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'json'
|
3
|
+
module Fluent
|
4
|
+
class OutputFieldMultiFlattenJson < Fluent::Output
|
5
|
+
Fluent::Plugin.register_output('field_flatten_json', self)
|
6
|
+
|
7
|
+
config_param :remove_tag_prefix, :string, :default => nil
|
8
|
+
config_param :add_tag_prefix, :string, :default => nil
|
9
|
+
config_param :flatten_key, :string, :default => nil
|
10
|
+
|
11
|
+
def emit(tag, es, chain)
|
12
|
+
tag = update_tag(tag)
|
13
|
+
es.each { |time, record|
|
14
|
+
Engine.emit(tag, time, flatten_field(record))
|
15
|
+
}
|
16
|
+
chain.next
|
17
|
+
end
|
18
|
+
|
19
|
+
def update_tag(tag)
|
20
|
+
if remove_tag_prefix
|
21
|
+
if remove_tag_prefix == tag
|
22
|
+
tag = ''
|
23
|
+
elsif tag.to_s.start_with?(remove_tag_prefix+'.')
|
24
|
+
tag = tag[remove_tag_prefix.length+1 .. -1]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
if add_tag_prefix
|
28
|
+
tag = tag && tag.length > 0 ? "#{add_tag_prefix}.#{tag}" : add_tag_prefix
|
29
|
+
end
|
30
|
+
return tag
|
31
|
+
end
|
32
|
+
|
33
|
+
def flatten(json, prefix)
|
34
|
+
json.keys.each do |key|
|
35
|
+
if prefix.empty?
|
36
|
+
full_path = key
|
37
|
+
else
|
38
|
+
full_path = [prefix, key].join('.')
|
39
|
+
end
|
40
|
+
|
41
|
+
if json[key].is_a?(Hash)
|
42
|
+
value = json[key]
|
43
|
+
json.delete key
|
44
|
+
json.merge! flatten(value, full_path)
|
45
|
+
else
|
46
|
+
value = json[key]
|
47
|
+
json.delete key
|
48
|
+
json[full_path] = value
|
49
|
+
end
|
50
|
+
end
|
51
|
+
return json
|
52
|
+
end
|
53
|
+
|
54
|
+
def flatten_field(record)
|
55
|
+
if !(flatten_key.nil?) and !(record.nil?) and record.key?(flatten_key)
|
56
|
+
source = record[flatten_key].to_s
|
57
|
+
json_field = JSON.parse(source)
|
58
|
+
res_hash = flatten(json_field,flatten_key)
|
59
|
+
res_hash.each do |key,val|
|
60
|
+
record[key] = val
|
61
|
+
end
|
62
|
+
end
|
63
|
+
return record
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
metadata
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-field-flatten-json
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Manoj Sharma
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-05-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: json
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Fluent output plugin for flattening a json field
|
56
|
+
email:
|
57
|
+
- vigyanik@gmail.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".gitignore"
|
63
|
+
- README.md
|
64
|
+
- Rakefile
|
65
|
+
- VERSION
|
66
|
+
- fluent-plugin-field-multiregex.gemspec
|
67
|
+
- lib/fluent/plugin/out_field_flatten_json.rb
|
68
|
+
homepage: https://github.com/CiscoZeus/fluent-plugin-field-flatten-json
|
69
|
+
licenses:
|
70
|
+
- MIT
|
71
|
+
metadata: {}
|
72
|
+
post_install_message:
|
73
|
+
rdoc_options: []
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
requirements: []
|
87
|
+
rubyforge_project:
|
88
|
+
rubygems_version: 2.5.1
|
89
|
+
signing_key:
|
90
|
+
specification_version: 4
|
91
|
+
summary: Fluent output plugin for flattening a json field
|
92
|
+
test_files: []
|