fluent-plugin-json-transform 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a43f517c612ec4d493e30ef71454a06bd7b94712
4
+ data.tar.gz: dfd9b372ecaf792ab38aa347552c694742accd39
5
+ SHA512:
6
+ metadata.gz: 6ef573459997fa3c02a3eae42aa8f5c719a59cbfad06d34f16b20373c59a15e56d94faecc175c40c8a3e7849833a6a064ff1024f015c32fcc726e60ff14a7d90
7
+ data.tar.gz: bf782fb72b29e33deec557b069cc9344be42abb8ea8949f09f0656c4010756ff81ff1d28fe78b30898d3ff89d07ee9ebf8c5c84dc96c02dfccd7813f9520e30a
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ .vagrant/
3
+ Vagrantfile
4
+ test.sh
@@ -0,0 +1,90 @@
1
+ # JSON Transform parser plugin for Fluentd
2
+
3
+ ##Overview
4
+ This is a [parser plugin](http://docs.fluentd.org/articles/parser-plugin-overview) for fluentd. It is **INCOMPATIBLE WITH FLUENTD v0.10.45 AND BELOW.**
5
+
6
+
7
+ It was created for the purpose of modifying [**good.js**](https://github.com/hapijs/good) logs
8
+ before storing them in Elasticsearch. It may not be useful for any other purpose, but be creative.
9
+
10
+ ##Installation
11
+ ```bash
12
+ git clone https://github.com/graysonc/fluent-plugin-json-transform
13
+ cd fluent-plugin-json-transform && gem build fluent-plugin-json-transform.gemspec
14
+ gem install fluent-plugin-json-transform
15
+ ```
16
+
17
+ ##Configuration
18
+ ```
19
+ <source>
20
+ type [tail|tcp|uydp|syslog|http] # or a custom input type which accepts the "format" parameter
21
+ format json_transform
22
+ transform_script [nothing|flatten|custom]
23
+ script_path "/home/grayson/transform_script.rb" # ignored if transform_script != custom
24
+ </source>
25
+ ```
26
+
27
+ `transform_script`: `nothing` to do nothing, `flatten` to flatten JSON by concatenating nested keys (see below), or `custom`
28
+
29
+ `script_path`: ignored if not using `custom` script. Point this to a Ruby script which implements the `JSONTransformer` class.
30
+
31
+ ###Flatten script
32
+ Flattens nested JSON by concatenating nested keys with '.'. Example:
33
+
34
+ ```
35
+ {
36
+ "hello": {
37
+ "world": true
38
+ },
39
+ "goodbye": {
40
+ "for": {
41
+ "now": true,
42
+ "ever": false
43
+ }
44
+ }
45
+ }
46
+ ```
47
+
48
+ Becomes
49
+
50
+ ```
51
+ {
52
+ "hello.world": true,
53
+ "goodbye.for.now": true,
54
+ "goodbye.for.ever": false
55
+ }
56
+ ```
57
+
58
+ ##Implementing JSONTransformer
59
+
60
+ The `JSONTransformer` class should have an instance method `transform` which takes a Ruby hash and returns a Ruby hash:
61
+
62
+ ```ruby
63
+ # lib/transform/flatten.rb
64
+ class JSONTransformer
65
+ def transform(json)
66
+ return flatten(json, "")
67
+ end
68
+
69
+ def flatten(json, prefix)
70
+ json.keys.each do |key|
71
+ if prefix.empty?
72
+ full_path = key
73
+ else
74
+ full_path = [prefix, key].join('.')
75
+ end
76
+
77
+ if json[key].is_a?(Hash)
78
+ value = json[key]
79
+ json.delete key
80
+ json.merge! flatten(value, full_path)
81
+ else
82
+ value = json[key]
83
+ json.delete key
84
+ json[full_path] = value
85
+ end
86
+ end
87
+ return json
88
+ end
89
+ end
90
+ ```
@@ -0,0 +1,20 @@
1
+ # coding: utf-8
2
+ Gem::Specification.new do |spec|
3
+ spec.name = "fluent-plugin-json-transform"
4
+ spec.version = "0.0.1"
5
+ spec.authors = ["Grayson Chao"]
6
+ spec.email = ["grayson.chao@gmail.com"]
7
+ spec.description = %q{Input parser plugin which allows arbitrary transformation of input JSON}
8
+ spec.summary = %q{Input parser plugin which allows arbitrary transformation of input JSON}
9
+ spec.homepage = "https://github.com/graysonc/fluent-plugin-json-transform"
10
+ spec.license = "MIT"
11
+
12
+ spec.files = `git ls-files`.split($/)
13
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
14
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
15
+ spec.require_paths = ["lib"]
16
+
17
+ spec.add_development_dependency "bundler", "~> 1.3"
18
+ spec.add_development_dependency "rake"
19
+ spec.add_development_dependency "rspec"
20
+ end
@@ -0,0 +1,32 @@
1
+ module Fluent
2
+ class TextParser
3
+ class JSONTransformParser
4
+ DEFAULTS = [ 'nothing', 'flatten' ]
5
+
6
+ include Configurable
7
+ config_param :transform_script, :string
8
+ config_param :script_path, :string
9
+
10
+ def configure(conf)
11
+ @transform_script = conf['transform_script']
12
+
13
+ if DEFAULTS.include?(@transform_script)
14
+ @transform_script =
15
+ "#{__dir__}/../../transform/#{@transform_script}.rb"
16
+ elsif @transform_script == 'custom'
17
+ @transform_script = conf['script_path']
18
+ end
19
+
20
+ require @transform_script
21
+ @transformer = JSONTransformer.new
22
+ end
23
+
24
+ def call(text)
25
+ raw_json = JSON.parse(text)
26
+ return nil, @transformer.transform(raw_json)
27
+ end
28
+ end
29
+ register_template("json_transform", Proc.new { JSONTransformParser.new })
30
+ end
31
+ end
32
+
@@ -0,0 +1,26 @@
1
+ class JSONTransformer
2
+ def transform(json)
3
+ return flatten(json, "")
4
+ end
5
+
6
+ def flatten(json, prefix)
7
+ json.keys.each do |key|
8
+ if prefix.empty?
9
+ full_path = key
10
+ else
11
+ full_path = [prefix, key].join('.')
12
+ end
13
+
14
+ if json[key].is_a?(Hash)
15
+ value = json[key]
16
+ json.delete key
17
+ json.merge! flatten(value, full_path)
18
+ else
19
+ value = json[key]
20
+ json.delete key
21
+ json[full_path] = value
22
+ end
23
+ end
24
+ return json
25
+ end
26
+ end
@@ -0,0 +1,5 @@
1
+ class JSONTransformer
2
+ def transform(json)
3
+ return json
4
+ end
5
+ end
metadata ADDED
@@ -0,0 +1,92 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-json-transform
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Grayson Chao
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-09-29 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.3'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '1.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: Input parser plugin which allows arbitrary transformation of input JSON
56
+ email:
57
+ - grayson.chao@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - .gitignore
63
+ - README.md
64
+ - fluent-plugin-json-transform.gemspec
65
+ - lib/fluent/plugin/parser_json_transform.rb
66
+ - lib/transform/flatten.rb
67
+ - lib/transform/nothing.rb
68
+ homepage: https://github.com/graysonc/fluent-plugin-json-transform
69
+ licenses:
70
+ - MIT
71
+ metadata: {}
72
+ post_install_message:
73
+ rdoc_options: []
74
+ require_paths:
75
+ - lib
76
+ required_ruby_version: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - '>='
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ required_rubygems_version: !ruby/object:Gem::Requirement
82
+ requirements:
83
+ - - '>='
84
+ - !ruby/object:Gem::Version
85
+ version: '0'
86
+ requirements: []
87
+ rubyforge_project:
88
+ rubygems_version: 2.0.14
89
+ signing_key:
90
+ specification_version: 4
91
+ summary: Input parser plugin which allows arbitrary transformation of input JSON
92
+ test_files: []