fluent-plugin-json-transform 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a43f517c612ec4d493e30ef71454a06bd7b94712
-  data.tar.gz: dfd9b372ecaf792ab38aa347552c694742accd39
+  metadata.gz: 36c342eeda778ad22180e136d3f489a8dc88f396
+  data.tar.gz: 0c1459d3f96f5d0c47ce96283714b3f398509667
 SHA512:
-  metadata.gz: 6ef573459997fa3c02a3eae42aa8f5c719a59cbfad06d34f16b20373c59a15e56d94faecc175c40c8a3e7849833a6a064ff1024f015c32fcc726e60ff14a7d90
-  data.tar.gz: bf782fb72b29e33deec557b069cc9344be42abb8ea8949f09f0656c4010756ff81ff1d28fe78b30898d3ff89d07ee9ebf8c5c84dc96c02dfccd7813f9520e30a
+  metadata.gz: c62858b9bad3b81f81cfa5e99919f8fa89adc2393ceb3c226827c16244b02a68e69285936a31b86458e5567d5ae7a1ccadfafed7ba1ab5368f89a79e9f1afc27
+  data.tar.gz: c1fa5de9c667ce0fbe34cf0ae5f655953fe31c5b12cbffae813a0de0ce7960e4b9a6ab45e072cc3441a53de57ca2b41542273679c5b18a93f705fb1c15bd2339

data/.gitignore

@@ -1,4 +1,4 @@
-*.gem
-.vagrant/
-Vagrantfile
-test.sh
+*.gem
+.vagrant/
+Vagrantfile
+test.sh

data/README.md

@@ -1,90 +1,99 @@
-# JSON Transform parser plugin for Fluentd
-
-##Overview
-This is a [parser plugin](http://docs.fluentd.org/articles/parser-plugin-overview) for fluentd. It is **INCOMPATIBLE WITH FLUENTD v0.10.45 AND BELOW.**
-
-
-It was created for the purpose of modifying [**good.js**](https://github.com/hapijs/good) logs
-before storing them in Elasticsearch. It may not be useful for any other purpose, but be creative.
-
-##Installation
-```bash
-git clone https://github.com/graysonc/fluent-plugin-json-transform
-cd fluent-plugin-json-transform && gem build fluent-plugin-json-transform.gemspec
-gem install fluent-plugin-json-transform
-```
-
-##Configuration
-```
-<source>
-  type [tail|tcp|uydp|syslog|http] # or a custom input type which accepts the "format" parameter
-  format json_transform
-  transform_script [nothing|flatten|custom]
-  script_path "/home/grayson/transform_script.rb" # ignored if transform_script != custom
-</source>
-```
-
-`transform_script`: `nothing` to do nothing, `flatten` to flatten JSON by concatenating nested keys (see below), or `custom` 
-
-`script_path`: ignored if not using `custom` script. Point this to a Ruby script which implements the `JSONTransformer` class.
-
-###Flatten script
-Flattens nested JSON by concatenating nested keys with '.'. Example:
-
-```
-{
-  "hello": {
-    "world": true
-  },
-  "goodbye": {
-    "for": {
-      "now": true,
-      "ever": false
-    }
-  }
-}
-```
-
-Becomes
-
-```
-{
-  "hello.world": true,
-  "goodbye.for.now": true,
-  "goodbye.for.ever": false
-}
-```
-
-##Implementing JSONTransformer
-
-The `JSONTransformer` class should have an instance method `transform` which takes a Ruby hash and returns a Ruby hash:
-
-```ruby
-# lib/transform/flatten.rb
-class JSONTransformer
-  def transform(json)
-    return flatten(json, "")
-  end
-
-  def flatten(json, prefix)
-    json.keys.each do |key|
-      if prefix.empty?
-        full_path = key
-      else
-        full_path = [prefix, key].join('.')
-      end
-
-      if json[key].is_a?(Hash)
-        value = json[key]
-        json.delete key
-        json.merge! flatten(value, full_path)
-      else
-        value = json[key]
-        json.delete key
-        json[full_path] = value
-      end
-    end
-    return json
-  end
-end
-```
+# JSON Transform parser plugin for Fluentd
+
+##Overview
+This is a [parser plugin](http://docs.fluentd.org/articles/parser-plugin-overview) for fluentd. It is **INCOMPATIBLE WITH FLUENTD v0.10.45 AND BELOW.**
+
+
+It was created for the purpose of modifying [**good.js**](https://github.com/hapijs/good) logs
+before storing them in Elasticsearch. It may not be useful for any other purpose, but be creative.
+
+##Installation
+```bash
+gem install fluent-plugin-json-transform
+```
+
+##Configuration
+```
+<source>
+  type [tail|tcp|udp|syslog|http] # or a custom input type which accepts the "format" parameter
+  format json_transform
+  transform_script [nothing|flatten|custom]
+  script_path "/home/grayson/transform_script.rb" # ignored if transform_script != custom
+</source>
+```
+
+`transform_script`: `nothing` to do nothing, `flatten` to flatten JSON by concatenating nested keys (see below), or `custom` 
+
+`script_path`: ignored if not using `custom` script. Point this to a Ruby script which implements the `JSONTransformer` class.
+
+###Flatten script
+Flattens nested JSON by concatenating nested keys with '.'. Example:
+
+```
+{
+  "hello": {
+    "world": true
+  },
+  "goodbye": {
+    "for": {
+      "now": true,
+      "ever": false
+    }
+  }
+}
+```
+
+Becomes
+
+```
+{
+  "hello.world": true,
+  "goodbye.for.now": true,
+  "goodbye.for.ever": false
+}
+```
+
+###New Filter Option
+Filtering is now supported, if you want to flatten your json after doing other parsing from the original source log.
+```
+<filter pattern>
+  @type json_transform
+  transform_script [nothing|flatten|custom]
+  script_path "/home/grayson/transform_script.rb" # ignored if transform_script != custom
+</filter>
+```
+
+
+##Implementing JSONTransformer
+
+The `JSONTransformer` class should have an instance method `transform` which takes a Ruby hash and returns a Ruby hash:
+
+```ruby
+# lib/transform/flatten.rb
+class JSONTransformer
+  def transform(json)
+    return flatten(json, "")
+  end
+
+  def flatten(json, prefix)
+    json.keys.each do |key|
+      if prefix.empty?
+        full_path = key
+      else
+        full_path = [prefix, key].join('.')
+      end
+
+      if json[key].is_a?(Hash)
+        value = json[key]
+        json.delete key
+        json.merge! flatten(value, full_path)
+      else
+        value = json[key]
+        json.delete key
+        json[full_path] = value
+      end
+    end
+    return json
+  end
+end
+```

data/fluent-plugin-json-transform.gemspec

@@ -1,20 +1,20 @@
-# coding: utf-8
-Gem::Specification.new do |spec|
-  spec.name          = "fluent-plugin-json-transform"
-  spec.version       = "0.0.1"
-  spec.authors       = ["Grayson Chao"]
-  spec.email         = ["grayson.chao@gmail.com"]
-  spec.description   = %q{Input parser plugin which allows arbitrary transformation of input JSON}
-  spec.summary       = %q{Input parser plugin which allows arbitrary transformation of input JSON}
-  spec.homepage      = "https://github.com/graysonc/fluent-plugin-json-transform"
-  spec.license       = "MIT"
-
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-end
+# coding: utf-8
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-json-transform"
+  spec.version       = "0.0.2"
+  spec.authors       = ["Grayson Chao","Matthew Jourard"]
+  spec.email         = ["grayson.chao@gmail.com","mjourard@gmail.com"]
+  spec.description   = %q{Input parser plugin which allows arbitrary transformation of input JSON}
+  spec.summary       = %q{Input parser plugin which allows arbitrary transformation of input JSON}
+  spec.homepage      = "https://github.com/mjourard/fluent-plugin-json-transform/"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/lib/fluent/plugin/filter_json_transform.rb

@@ -0,0 +1,29 @@
+module Fluent
+  class JSONTransformFilter < Filter
+    Fluent::Plugin.register_filter('json_transform', self)
+
+    DEFAULTS = [ 'nothing', 'flatten' ]
+
+    include Configurable
+    config_param :transform_script, :string
+    config_param :script_path, :string
+
+    def configure(conf)
+      @transform_script = conf['transform_script']
+
+      if DEFAULTS.include?(@transform_script)
+        @transform_script = "#{__dir__}/../../transform/#{@transform_script}.rb"
+      elsif @transform_script == 'custom'
+        @transform_script = conf['script_path']
+      end
+
+      require @transform_script
+      @transformer = JSONTransformer.new
+    end
+
+    def filter(tag, time, record)
+      flattened = @transformer.transform(record)
+      return flattened
+    end
+  end
+end

data/lib/fluent/plugin/parser_json_transform.rb

@@ -1,32 +1,37 @@
-module Fluent
-  class TextParser
-    class JSONTransformParser
-      DEFAULTS = [ 'nothing', 'flatten' ]
-
-      include Configurable
-      config_param :transform_script, :string
-      config_param :script_path, :string
-
-      def configure(conf)
-        @transform_script = conf['transform_script']
-
-        if DEFAULTS.include?(@transform_script)
-          @transform_script =
-            "#{__dir__}/../../transform/#{@transform_script}.rb"
-        elsif @transform_script == 'custom'
-          @transform_script = conf['script_path']
-        end
-
-        require @transform_script
-        @transformer = JSONTransformer.new
-      end
-
-      def call(text)
-        raw_json = JSON.parse(text)
-        return nil, @transformer.transform(raw_json)
-      end
-    end
-  register_template("json_transform", Proc.new { JSONTransformParser.new })
-  end
-end
-
+module Fluent
+  class TextParser
+    class JSONTransformParser
+      DEFAULTS = [ 'nothing', 'flatten' ]
+
+      include Configurable
+      config_param :transform_script, :string
+      config_param :script_path, :string
+
+      def configure(conf)
+        @transform_script = conf['transform_script']
+
+        if DEFAULTS.include?(@transform_script)
+          @transform_script =
+            "#{__dir__}/../../transform/#{@transform_script}.rb"
+        elsif @transform_script == 'custom'
+          @transform_script = conf['script_path']
+        end
+
+        require @transform_script
+        @transformer = JSONTransformer.new
+      end
+
+      def call(text)
+        raw_json = JSON.parse(text)
+        return nil, @transformer.transform(raw_json)
+      end
+
+      def parse(text)
+        raw_json = JSON.parse(text)
+        return nil, @transformer.transform(raw_json)
+      end
+    end
+  register_template("json_transform", Proc.new { JSONTransformParser.new })
+  end
+end
+

data/lib/transform/flatten.rb

@@ -1,26 +1,26 @@
-class JSONTransformer
-  def transform(json)
-    return flatten(json, "")
-  end
-
-  def flatten(json, prefix)
-    json.keys.each do |key|
-      if prefix.empty?
-        full_path = key
-      else
-        full_path = [prefix, key].join('.')
-      end
-
-      if json[key].is_a?(Hash)
-        value = json[key]
-        json.delete key
-        json.merge! flatten(value, full_path)
-      else
-        value = json[key]
-        json.delete key
-        json[full_path] = value
-      end
-    end
-    return json
-  end
-end
+class JSONTransformer
+  def transform(json)
+    return flatten(json, "")
+  end
+
+  def flatten(json, prefix)
+    json.keys.each do |key|
+      if prefix.empty?
+        full_path = key
+      else
+        full_path = [prefix, key].join('.')
+      end
+
+      if json[key].is_a?(Hash)
+        value = json[key]
+        json.delete key
+        json.merge! flatten(value, full_path)
+      else
+        value = json[key]
+        json.delete key
+        json[full_path] = value
+      end
+    end
+    return json
+  end
+end

data/lib/transform/nothing.rb

@@ -1,5 +1,5 @@
-class JSONTransformer
-  def transform(json)
-    return json
-  end
-end
+class JSONTransformer
+  def transform(json)
+    return json
+  end
+end

metadata

@@ -1,71 +1,74 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-json-transform
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Grayson Chao
+- Matthew Jourard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-29 00:00:00.000000000 Z
+date: 2016-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Input parser plugin which allows arbitrary transformation of input JSON
 email:
 - grayson.chao@gmail.com
+- mjourard@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - README.md
 - fluent-plugin-json-transform.gemspec
+- lib/fluent/plugin/filter_json_transform.rb
 - lib/fluent/plugin/parser_json_transform.rb
 - lib/transform/flatten.rb
 - lib/transform/nothing.rb
-homepage: https://github.com/graysonc/fluent-plugin-json-transform
+homepage: https://github.com/mjourard/fluent-plugin-json-transform/
 licenses:
 - MIT
 metadata: {}
@@ -75,17 +78,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Input parser plugin which allows arbitrary transformation of input JSON