fluent-plugin-json-transform_ex 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 700d72367418f0f7cf2630152aff119cb796e17fcf6aedd25462b73d76893853
+  data.tar.gz: 4f95f6f62c7a7de3d7004f3b1c2a86c556edf2bf2210c5a9c9e67fd7986eceb6
+SHA512:
+  metadata.gz: 943274519c5c26f54c57419e2be56b61d9483eae83c821e66dea683a32f9edda3d40b61cdbc068442ec1a5349e17855ea04b62322ad528f6de83970857769903
+  data.tar.gz: 5fcc73c15e2cd92a964639e876d91c9e9e479e74b657d527b5f424695cf8fda40f05a0a9c2ae856cbd062d6cad9eafe28ec2ac2bd2fe8a95992cebf48702089e

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.vagrant/
+Vagrantfile
+test.sh
+.gitconfig

data/README.md

@@ -0,0 +1,103 @@
+# JSON Transform parser plugin for Fluentd
+
+## Overview
+This is a [parser plugin](http://docs.fluentd.org/articles/parser-plugin-overview) for fluentd. It is **INCOMPATIBLE WITH FLUENTD v0.10.45 AND BELOW.**
+
+
+It was created for the purpose of modifying [**good.js**](https://github.com/hapijs/good) logs
+before storing them in Elasticsearch. It may not be useful for any other purpose, but be creative.
+
+## Installation
+```bash
+gem install fluent-plugin-json-transform_ex --version 0.0.1
+```
+
+## Configuration
+```
+<source>
+  type [tail|tcp|udp|syslog|http] # or a custom input type which accepts the "format" parameter
+  format json_transform
+  transform_script [nothing|flatten|custom]
+  script_path "/home/grayson/transform_script.rb" # ignored if transform_script != custom
+  class_name "CustomJSONTransformer" # [optional] default value is "JSONTransformer", ignored if transform_script != custom
+</source>
+```
+
+`transform_script`: `nothing` to do nothing, `flatten` to flatten JSON by concatenating nested keys (see below), or `custom` 
+
+`script_path`: ignored if not using `custom` script. Point this to a Ruby script which implements the class from `class_name` parameter.
+
+`class_name`: [optional] ignored if not using `custom` script. Define name of a class which is used for transformation in `script_path` script.
+
+### Flatten script
+Flattens nested JSON by concatenating nested keys with '.'. Example:
+
+```
+{
+  "hello": {
+    "world": true
+  },
+  "goodbye": {
+    "for": {
+      "now": true,
+      "ever": false
+    }
+  }
+}
+```
+
+Becomes
+
+```
+{
+  "hello.world": true,
+  "goodbye.for.now": true,
+  "goodbye.for.ever": false
+}
+```
+
+### Filter Option
+If you want to flatten your json after doing other parsing from the original source log.
+```
+<filter pattern>
+  @type json_transform
+  transform_script [nothing|flatten|custom]
+  script_path "/home/grayson/transform_script.rb" # ignored if transform_script != custom
+  class_name "CustomJSONTransformer" # [optional] default value is "JSONTransformer", ignored if transform_script != custom
+</filter>
+```
+
+
+## Implementing transformer class
+
+The transformer class should have an instance method `transform` which takes a Ruby hash and returns a Ruby hash. Pay attention that the name of a class should be the same as you defined in `class_name` parameter or `JSONTransformer` in case `class_name` parameter is not defined:
+
+```ruby
+# lib/transform/flatten.rb
+class JSONTransformer # or any class name defined in class_name parameter
+  def transform(json)
+    return flatten(json, "")
+  end
+
+  def flatten(json, prefix)
+    json.keys.each do |key|
+      if prefix.empty?
+        full_path = key
+      else
+        full_path = [prefix, key].join('.')
+      end
+
+      if json[key].is_a?(Hash)
+        value = json[key]
+        json.delete key
+        json.merge! flatten(value, full_path)
+      else
+        value = json[key]
+        json.delete key
+        json[full_path] = value
+      end
+    end
+    return json
+  end
+end
+```

data/fluent-plugin-json-transform_ex.gemspec

@@ -0,0 +1,20 @@
+# coding: utf-8
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-json-transform_ex"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Grayson Chao", "Yevhen Fabizhevskyi"]
+  spec.email         = ["grayson.chao@gmail.com", "fabasoad@gmail.com"]
+  spec.description   = %q{Input parser plugin which allows arbitrary transformation of input JSON}
+  spec.summary       = %q{Input parser plugin which allows arbitrary transformation of input JSON}
+  spec.homepage      = "https://github.com/fabasoad/fluent-plugin-json-transform_ex"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/lib/fluent/plugin/filter_json_transform_ex.rb

@@ -0,0 +1,36 @@
+module Fluent
+  class JSONTransformFilter < Filter
+    Fluent::Plugin.register_filter('json_transform_ex', self)
+
+    DEFAULTS = [ 'nothing', 'flatten' ]
+    DEFAULT_CLASS_NAME = 'JSONTransformer'
+
+    include Configurable
+    config_param :transform_script, :string
+    config_param :script_path, :string
+    config_param :class_name, :string
+
+    def configure(conf)
+      @transform_script = conf['transform_script']
+
+      if DEFAULTS.include?(@transform_script)
+        @transform_script = "#{__dir__}/../../transform/#{@transform_script}.rb"
+        className = DEFAULT_CLASS_NAME
+      elsif @transform_script == 'custom'
+        @transform_script = conf['script_path']
+        className = conf['class_name'] || DEFAULT_CLASS_NAME
+      end
+
+      require @transform_script
+      begin
+        @transformer = Object.const_get(className).new
+      rescue NameError
+        @transformer = Object.const_get(DEFAULT_CLASS_NAME).new
+      end
+    end
+
+    def filter(tag, time, record)
+      return @transformer.transform(record)
+    end
+  end
+end

data/lib/fluent/plugin/parser_json_transform_ex.rb

@@ -0,0 +1,44 @@
+module Fluent
+  class TextParser
+    class JSONTransformParser
+      DEFAULTS = [ 'nothing', 'flatten' ]
+      DEFAULT_CLASS_NAME = 'JSONTransformer'
+
+      include Configurable
+      config_param :transform_script, :string
+      config_param :script_path, :string
+      config_param :class_name, :string
+
+      def configure(conf)
+        @transform_script = conf['transform_script']
+
+        if DEFAULTS.include?(@transform_script)
+          @transform_script = "#{__dir__}/../../transform/#{@transform_script}.rb"
+          className = DEFAULT_CLASS_NAME
+        elsif @transform_script == 'custom'
+          @transform_script = conf['script_path']
+          className = conf['class_name'] || DEFAULT_CLASS_NAME
+        end
+
+        require @transform_script
+        begin
+          @transformer = Object.const_get(className).new
+        rescue NameError
+          @transformer = Object.const_get(DEFAULT_CLASS_NAME).new
+        end
+      end
+
+      def call(text)
+        raw_json = JSON.parse(text)
+        return nil, @transformer.transform(raw_json)
+      end
+
+      def parse(text)
+        raw_json = JSON.parse(text)
+        return nil, @transformer.transform(raw_json)
+      end
+    end
+  register_template("json_transform_ex", Proc.new { JSONTransformParser.new })
+  end
+end
+

data/lib/transform/flatten.rb

@@ -0,0 +1,26 @@
+class JSONTransformer
+  def transform(json)
+    return flatten(json, "")
+  end
+
+  def flatten(json, prefix)
+    json.keys.each do |key|
+      if prefix.empty?
+        full_path = key
+      else
+        full_path = [prefix, key].join('.')
+      end
+
+      if json[key].is_a?(Hash)
+        value = json[key]
+        json.delete key
+        json.merge! flatten(value, full_path)
+      else
+        value = json[key]
+        json.delete key
+        json[full_path] = value
+      end
+    end
+    return json
+  end
+end

data/lib/transform/nothing.rb

@@ -0,0 +1,5 @@
+class JSONTransformer
+  def transform(json)
+    return json
+  end
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-json-transform_ex
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Grayson Chao
+- Yevhen Fabizhevskyi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Input parser plugin which allows arbitrary transformation of input JSON
+email:
+- grayson.chao@gmail.com
+- fabasoad@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- README.md
+- fluent-plugin-json-transform_ex.gemspec
+- lib/fluent/plugin/filter_json_transform_ex.rb
+- lib/fluent/plugin/parser_json_transform_ex.rb
+- lib/transform/flatten.rb
+- lib/transform/nothing.rb
+homepage: https://github.com/fabasoad/fluent-plugin-json-transform_ex
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Input parser plugin which allows arbitrary transformation of input JSON
+test_files: []