fluent-plugin-remove-empty 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c27b47de51f0e72ef214b0d5c1951bd9922168d5
+  data.tar.gz: 2601881938c556aa5b6478ab1363afb12d1abfa8
+SHA512:
+  metadata.gz: 1dc1429b3686c5f6c498f202a57dc9b09fc15f443880b0623602aac247e0eee5ee3dddccbd2976dddc2b3db79a38415fb73dc494ca348e465d9207a2e825afb7
+  data.tar.gz: ef9cb7638d5c22438bae04da151efc50183d438357b10417ca7ee2de77ee981eae879ad1c96dd01fd4958a2a872fe4f5e99ddfc14f30a90b757cabda1bee0380

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,9 @@
+# fluent-plugin-remove-empty
+
+Fluentd plugin to remove empty fields of a event record
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-remove-empty

data/fluent-plugin-remove-empty.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-remove-empty"
+  gem.version     = "0.0.1"
+  gem.authors     = ["Chadin Anuwattanaporn"]
+  gem.email       = "chadin@sogamo.com"
+  gem.homepage    = "https://github.com/newcmd001/fluent-plugin-remove-empty"
+  gem.description = "Fluentd plugin to remove empty fields of a event record"
+  gem.summary     = gem.description
+  gem.licenses    = ["MIT"]
+  gem.has_rdoc    = false
+
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
+end

data/lib/fluent/plugin/out_remove_empty.rb

@@ -0,0 +1,176 @@
+require 'socket'
+require 'ostruct'
+
+module Fluent
+  class RemoveEmptyOutput < Output
+    Fluent::Plugin.register_output('remove_empty', self)
+
+    def initialize
+      super
+    end
+
+    config_param :tag, :string, :default => nil
+
+    BUILTIN_CONFIGURATIONS = %W(type tag)
+
+    # To support log_level option implemented by Fluentd v0.10.43
+    unless method_defined?(:log)
+      define_method("log") { $log }
+    end
+
+    def configure(conf)
+      super
+
+      @map = {}
+      conf.each_pair { |k, v|
+        next if BUILTIN_CONFIGURATIONS.include?(k)
+        conf.has_key?(k) # to suppress unread configuration warning
+        @map[k] = v
+      }
+
+      if @tag.nil?
+        raise Fluent::ConfigError, "out_remove_empty: `tag` must be specified"
+      end
+      
+      @placeholder_expander =
+      if @enable_ruby
+          # require utilities which would be used in ruby placeholders
+          require 'pathname'
+          require 'uri'
+          require 'cgi'
+          RubyPlaceholderExpander.new(log)
+          else
+          PlaceholderExpander.new(log)
+      end
+      
+      @hostname = Socket.gethostname
+    end
+
+    def emit(tag, es, chain)
+      tag_parts = tag.split('.')
+      tag_prefix = tag_prefix(tag_parts)
+      tag_suffix = tag_suffix(tag_parts)
+      placeholders = {
+        'tag' => tag,
+        'tags' => tag_parts,
+        'tag_parts' => tag_parts,
+        'tag_prefix' => tag_prefix,
+        'tag_suffix' => tag_suffix,
+        'hostname' => @hostname,
+      }
+      last_record = nil
+      es.each {|time, record|
+        last_record = record # for debug log
+        new_tag, new_record = reform(@tag, time, record, placeholders)
+        Engine.emit(new_tag, time, new_record)
+      }
+      chain.next
+    rescue => e
+      log.warn "remove_empty: #{e.class} #{e.message} #{e.backtrace.first}"
+      log.debug "remove_empty: tag:#{@tag} map:#{@map} record:#{last_record} placeholders:#{placeholders}"
+    end
+
+    private
+
+    def reform(tag, time, record, opts)
+      @placeholder_expander.prepare_placeholders(time, record, opts)
+      new_tag = @placeholder_expander.expand(tag)
+
+      new_record = record.dup
+      new_record.each_key do |k|
+        if new_record[k] == ""
+          new_record.delete(k)
+        end
+      end
+
+      [new_tag, new_record]
+    end
+
+    def tag_prefix(tag_parts)
+      return [] if tag_parts.empty?
+      tag_prefix = [tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        tag_prefix[i] = "#{tag_prefix[i-1]}.#{tag_parts[i]}"
+      end
+      tag_prefix
+    end
+
+    def tag_suffix(tag_parts)
+      return [] if tag_parts.empty?
+      rev_tag_parts = tag_parts.reverse
+      rev_tag_suffix = [rev_tag_parts.first]
+      1.upto(tag_parts.size-1).each do |i|
+        rev_tag_suffix[i] = "#{rev_tag_parts[i]}.#{rev_tag_suffix[i-1]}"
+      end
+      rev_tag_suffix.reverse
+    end
+
+    class PlaceholderExpander
+      attr_reader :placeholders, :log
+
+      def initialize(log)
+        @log = log
+      end
+
+      def prepare_placeholders(time, record, opts)
+        placeholders = { '${time}' => Time.at(time).to_s }
+        record.each {|key, value| placeholders.store("${#{key}}", value) }
+
+        opts.each do |key, value|
+          if value.kind_of?(Array) # tag_parts, etc
+            size = value.size
+            value.each_with_index { |v, idx|
+              placeholders.store("${#{key}[#{idx}]}", v)
+              placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
+            }
+          else # string, interger, float, and others?
+            placeholders.store("${#{key}}", value)
+          end
+        end
+
+        @placeholders = placeholders
+      end
+
+      def expand(str)
+        str.gsub(/(\${[a-z_]+(\[-?[0-9]+\])?}|__[A-Z_]+__)/) {
+          log.warn "remove_empty: unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
+          @placeholders[$1]
+        }
+      end
+    end
+
+    class RubyPlaceholderExpander
+      attr_reader :placeholders, :log
+
+      def initialize(log)
+        @log = log
+      end
+
+      # Get placeholders as a struct
+      #
+      # @param [Time]   time        the time
+      # @param [Hash]   record      the record
+      # @param [Hash]   opts        others
+      def prepare_placeholders(time, record, opts)
+        struct = UndefOpenStruct.new(record)
+        struct.time = Time.at(time)
+        opts.each {|key, value| struct.__send__("#{key}=", value) }
+        @placeholders = struct
+      end
+
+      # Replace placeholders in a string
+      #
+      # @param [String] str         the string to be replaced
+      def expand(str)
+        str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
+        eval "\"#{str}\"", @placeholders.instance_eval { binding }
+      end
+
+      class UndefOpenStruct < OpenStruct
+        (Object.instance_methods).each do |m|
+          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-remove-empty
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Chadin Anuwattanaporn
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to remove empty fields of a event record
+email: chadin@sogamo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- fluent-plugin-remove-empty.gemspec
+- lib/fluent/plugin/out_remove_empty.rb
+homepage: https://github.com/newcmd001/fluent-plugin-remove-empty
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to remove empty fields of a event record
+test_files: []