fluent-plugin-json-transform 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a43f517c612ec4d493e30ef71454a06bd7b94712
+  data.tar.gz: dfd9b372ecaf792ab38aa347552c694742accd39
+SHA512:
+  metadata.gz: 6ef573459997fa3c02a3eae42aa8f5c719a59cbfad06d34f16b20373c59a15e56d94faecc175c40c8a3e7849833a6a064ff1024f015c32fcc726e60ff14a7d90
+  data.tar.gz: bf782fb72b29e33deec557b069cc9344be42abb8ea8949f09f0656c4010756ff81ff1d28fe78b30898d3ff89d07ee9ebf8c5c84dc96c02dfccd7813f9520e30a

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.vagrant/
+Vagrantfile
+test.sh

data/README.md

@@ -0,0 +1,90 @@
+# JSON Transform parser plugin for Fluentd
+
+##Overview
+This is a [parser plugin](http://docs.fluentd.org/articles/parser-plugin-overview) for fluentd. It is **INCOMPATIBLE WITH FLUENTD v0.10.45 AND BELOW.**
+
+
+It was created for the purpose of modifying [**good.js**](https://github.com/hapijs/good) logs
+before storing them in Elasticsearch. It may not be useful for any other purpose, but be creative.
+
+##Installation
+```bash
+git clone https://github.com/graysonc/fluent-plugin-json-transform
+cd fluent-plugin-json-transform && gem build fluent-plugin-json-transform.gemspec
+gem install fluent-plugin-json-transform
+```
+
+##Configuration
+```
+<source>
+  type [tail|tcp|uydp|syslog|http] # or a custom input type which accepts the "format" parameter
+  format json_transform
+  transform_script [nothing|flatten|custom]
+  script_path "/home/grayson/transform_script.rb" # ignored if transform_script != custom
+</source>
+```
+
+`transform_script`: `nothing` to do nothing, `flatten` to flatten JSON by concatenating nested keys (see below), or `custom` 
+
+`script_path`: ignored if not using `custom` script. Point this to a Ruby script which implements the `JSONTransformer` class.
+
+###Flatten script
+Flattens nested JSON by concatenating nested keys with '.'. Example:
+
+```
+{
+  "hello": {
+    "world": true
+  },
+  "goodbye": {
+    "for": {
+      "now": true,
+      "ever": false
+    }
+  }
+}
+```
+
+Becomes
+
+```
+{
+  "hello.world": true,
+  "goodbye.for.now": true,
+  "goodbye.for.ever": false
+}
+```
+
+##Implementing JSONTransformer
+
+The `JSONTransformer` class should have an instance method `transform` which takes a Ruby hash and returns a Ruby hash:
+
+```ruby
+# lib/transform/flatten.rb
+class JSONTransformer
+  def transform(json)
+    return flatten(json, "")
+  end
+
+  def flatten(json, prefix)
+    json.keys.each do |key|
+      if prefix.empty?
+        full_path = key
+      else
+        full_path = [prefix, key].join('.')
+      end
+
+      if json[key].is_a?(Hash)
+        value = json[key]
+        json.delete key
+        json.merge! flatten(value, full_path)
+      else
+        value = json[key]
+        json.delete key
+        json[full_path] = value
+      end
+    end
+    return json
+  end
+end
+```

data/fluent-plugin-json-transform.gemspec

@@ -0,0 +1,20 @@
+# coding: utf-8
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-json-transform"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Grayson Chao"]
+  spec.email         = ["grayson.chao@gmail.com"]
+  spec.description   = %q{Input parser plugin which allows arbitrary transformation of input JSON}
+  spec.summary       = %q{Input parser plugin which allows arbitrary transformation of input JSON}
+  spec.homepage      = "https://github.com/graysonc/fluent-plugin-json-transform"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/lib/fluent/plugin/parser_json_transform.rb

@@ -0,0 +1,32 @@
+module Fluent
+  class TextParser
+    class JSONTransformParser
+      DEFAULTS = [ 'nothing', 'flatten' ]
+
+      include Configurable
+      config_param :transform_script, :string
+      config_param :script_path, :string
+
+      def configure(conf)
+        @transform_script = conf['transform_script']
+
+        if DEFAULTS.include?(@transform_script)
+          @transform_script =
+            "#{__dir__}/../../transform/#{@transform_script}.rb"
+        elsif @transform_script == 'custom'
+          @transform_script = conf['script_path']
+        end
+
+        require @transform_script
+        @transformer = JSONTransformer.new
+      end
+
+      def call(text)
+        raw_json = JSON.parse(text)
+        return nil, @transformer.transform(raw_json)
+      end
+    end
+  register_template("json_transform", Proc.new { JSONTransformParser.new })
+  end
+end
+

data/lib/transform/flatten.rb

@@ -0,0 +1,26 @@
+class JSONTransformer
+  def transform(json)
+    return flatten(json, "")
+  end
+
+  def flatten(json, prefix)
+    json.keys.each do |key|
+      if prefix.empty?
+        full_path = key
+      else
+        full_path = [prefix, key].join('.')
+      end
+
+      if json[key].is_a?(Hash)
+        value = json[key]
+        json.delete key
+        json.merge! flatten(value, full_path)
+      else
+        value = json[key]
+        json.delete key
+        json[full_path] = value
+      end
+    end
+    return json
+  end
+end

data/lib/transform/nothing.rb

@@ -0,0 +1,5 @@
+class JSONTransformer
+  def transform(json)
+    return json
+  end
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-json-transform
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Grayson Chao
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Input parser plugin which allows arbitrary transformation of input JSON
+email:
+- grayson.chao@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- fluent-plugin-json-transform.gemspec
+- lib/fluent/plugin/parser_json_transform.rb
+- lib/transform/flatten.rb
+- lib/transform/nothing.rb
+homepage: https://github.com/graysonc/fluent-plugin-json-transform
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Input parser plugin which allows arbitrary transformation of input JSON
+test_files: []