fluent-plugin-field-flatten-json 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b08b1dc98e401ecd654c444f2abd4ef5d82258e2
+  data.tar.gz: 8cba69a2ed39dfbf591868cca3d0b4c50b25be94
+SHA512:
+  metadata.gz: 7eef2a29058f5afa3ce5eb820ce74a8f62e1bed08d11f62769032beb81a9066d81fd24a8e12a845572f8ee15a44d5750c76af1f1ac56a94541135aaf0f1a4e5f
+  data.tar.gz: 38155e5f519b13fdf040740b059a7b6bc671ef8ab82bad7650cd23f4a5e443d3f5fb94b6533a264669f03d92ffae382442f5e5abbb0141001ab08c3ffa62847e

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/README.md

@@ -0,0 +1,20 @@
+fluent-plugin-field-flatten-json
+===========================
+
+Fluent output plugin for reforming a record using multiple named capture regular expressions
+
+## Installation
+
+Use RubyGems:
+
+    td-agent-gem install fluent-plugin-field-flatten-json
+
+## Configuration
+
+    <match pattern>
+        type                field_flatten_json
+        flatten_key         message
+        remove_tag_prefix   raw
+        add_tag_prefix      parsed
+    </match>
+

data/Rakefile

@@ -0,0 +1,6 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_tasks"
+Bundler::GemHelper.install_tasks
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/fluent-plugin-field-multiregex.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-field-flatten-json"
+  gem.description   = "Fluent output plugin for flattening a json field"
+  gem.homepage      = "https://github.com/CiscoZeus/fluent-plugin-field-flatten-json"
+  gem.summary       = gem.description
+  gem.version       = File.read("VERSION").strip
+  gem.authors       = ["Manoj Sharma"]
+  gem.email         = ["vigyanik@gmail.com"]
+  gem.has_rdoc      = false
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd"
+  gem.add_dependency "json"
+  gem.add_development_dependency "rake"
+end
+

data/lib/fluent/plugin/out_field_flatten_json.rb

@@ -0,0 +1,66 @@
+require 'rubygems'
+require 'json'
+module Fluent
+  class OutputFieldMultiFlattenJson < Fluent::Output
+    Fluent::Plugin.register_output('field_flatten_json', self)
+
+    config_param :remove_tag_prefix,  :string, :default => nil
+    config_param :add_tag_prefix,     :string, :default => nil
+    config_param :flatten_key,        :string, :default => nil
+
+    def emit(tag, es, chain)
+      tag = update_tag(tag)
+      es.each { |time, record|
+        Engine.emit(tag, time, flatten_field(record))
+      }
+      chain.next
+    end
+
+    def update_tag(tag)
+      if remove_tag_prefix
+        if remove_tag_prefix == tag
+          tag = ''
+        elsif tag.to_s.start_with?(remove_tag_prefix+'.')
+          tag = tag[remove_tag_prefix.length+1 .. -1]
+        end
+      end
+      if add_tag_prefix
+        tag = tag && tag.length > 0 ? "#{add_tag_prefix}.#{tag}" : add_tag_prefix
+      end
+      return tag
+    end
+
+    def flatten(json, prefix)
+      json.keys.each do |key|
+        if prefix.empty?
+          full_path = key
+        else
+          full_path = [prefix, key].join('.')
+        end
+  
+        if json[key].is_a?(Hash)
+          value = json[key]
+          json.delete key
+          json.merge! flatten(value, full_path)
+        else
+          value = json[key]
+          json.delete key
+          json[full_path] = value
+        end
+      end
+      return json
+    end
+
+    def flatten_field(record)
+      if !(flatten_key.nil?) and !(record.nil?) and record.key?(flatten_key)
+        source = record[flatten_key].to_s
+        json_field = JSON.parse(source)
+        res_hash = flatten(json_field,flatten_key)
+        res_hash.each do |key,val|
+          record[key] = val
+        end
+      end
+      return record
+    end
+  end
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-field-flatten-json
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Manoj Sharma
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent output plugin for flattening a json field
+email:
+- vigyanik@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-field-multiregex.gemspec
+- lib/fluent/plugin/out_field_flatten_json.rb
+homepage: https://github.com/CiscoZeus/fluent-plugin-field-flatten-json
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Fluent output plugin for flattening a json field
+test_files: []