fluent-plugin-data-rejecter 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 422a4edcad8c5e9645e5da2a922af480ff148f15
+  data.tar.gz: 0d83f59a3da8dd460d4210dcee89206d508b7375
+SHA512:
+  metadata.gz: 9744819ba2a0ab404e0058f90f827adeba79d89924b341b2f45622330abea24319bb776fa2d3bf436cb6a2887b20dfb9b85fce5d674859a9e619753d265b1f89
+  data.tar.gz: 8ed81b35b5433bc0dff08ceb3bf12365b29b9ebea4d73d1c624c05ae1d8f527203446bbf6f84dd73d2c833947d3d8ea2859c4b3509921ce6e8617ed94acfa88d

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 hirotaka tajiri
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+#fluent-plugin-data-rejecter
+
+## Overview
+
+Output plugin to reject key pairs
+
+* reject key pairs
+* remove tag prefix
+* add tag prefix
+
+## Parameters
+
+- remove_prefix
+
+    remove tag if tag matches this pattern
+
+- add_prefix
+
+    add tag for re-emit.
+
+- reject_keys
+
+    reject key pair for re-emit.
+
+## Configuration
+
+Example:
+
+    <match abc.def.**>
+        type          data_rejecter
+        remove_prefix abc
+        add_prefix    123
+        reject_keys   key key1
+    </match>
+
+input
+
+    abc.def.tag: {"dat":"message", "key":"value", "key1":"value2", "key2":"value2", ....}
+
+output
+
+    123.def.tag: {"dat":"message", "key2":"value2", ....}
+
+### remove_prefix (complete matching)
+
+config  |  tag  #=> result
+
+    abc  | abc.def.tag #=> removed
+    abc. | abc.def.tag #=> removed
+    ab   | abc.def.tag #=> not removed
+
+## Copyright
+
+Copyright (c) 2014 Hirotaka Tajiri. See [LICENSE](LICENSE.txt) for details.

data/Rakefile

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do | spec |
+    spec.pattern = FileList['spec/*_spec.rb']
+end
+task :default => :spec

data/fluent-plugin-data-rejecter.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do | gs |
+    gs.name          = "fluent-plugin-data-rejecter"
+    gs.version       = "1.0.0"
+    gs.authors       = ["Hirotaka Tajiri"]
+    gs.email         = "ganryu_koziro@excite.co.jp"
+    gs.homepage      = "https://github.com/hirotaka-tajiri/fluent-plugin-data-rejecter"
+    gs.description   = "Fluent plugin Output filer to reject key pair"
+    gs.summary       = "Output filter plugin to reject key pair"
+    gs.licenses      = ["MIT"]
+    gs.has_rdoc      = false
+
+    gs.files         = `git ls-files`.split("\n")
+    gs.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+    gs.executables   = `git ls-files -- bin/*`.split("\n").map{| f | File.basename(f)}
+    gs.require_paths = ['lib']
+
+    gs.add_dependency 'fluentd', "~> 0.10"
+    gs.add_development_dependency "rake", '~>0'
+    gs.add_development_dependency "rspec", '~>0'
+end

data/lib/fluent/plugin/out_data_rejecter.rb

@@ -0,0 +1,32 @@
+module Fluent
+    class DataRejecterOutput < Output
+        Fluent::Plugin.register_output('data_rejecter', self)
+
+        config_param :remove_prefix, :string, :default => ''
+        config_param :add_prefix,    :string, :default => ''
+        config_param :reject_keys,   :string, :default => ''
+
+        def configure(conf)
+            super
+            @remove = @remove_prefix.end_with?('.') ? @remove_prefix : @remove_prefix + '.'
+            @add    = @add_prefix.end_with?('.')    ? @add_prefix    : @add_prefix    + '.'
+            @add_p  = @add.chop
+        end
+
+        def emit(tag, es, chain)
+            tmptag = (tag[@remove.size .. -1] if tag.start_with?(@remove)) || (tag == @remove_prefix ? '' : tag)
+            tag    = @add_p.size.zero? ? (tmptag.size.zero? ? "data_rejecter.tag_lost" : tmptag) : (tmptag.size.zero? ? @add_p : @add + tmptag)
+
+            es.each {| time, record |
+                Engine.emit(tag, time, reject_record(record))
+            }
+            chain.next
+        end
+
+      private
+        def reject_record(record)
+            @reject_keys.split.each{| v | record.delete(v) if record.has_key?(v)}
+            record
+        end
+    end
+end

data/spec/out_data_rejecter_spec.rb

@@ -0,0 +1,270 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+describe Fluent::DataRejecterOutput do
+
+    before { Fluent::Test.setup }
+
+    def create_driver(conf, tag='default.tag')
+        Fluent::Test::OutputTestDriver.new(Fluent::DataRejecterOutput, tag).configure(conf)
+    end
+
+    let(:time_now) { Time.now }
+
+    describe 'test config ' do
+
+        it 'Check remove prefix AND add prefix AND reject_keys2' do
+            d = create_driver %[
+               remove_prefix abc
+               add_prefix    123
+               reject_keys   key1 key2
+            ]
+            d.instance.inspect
+            expect(d.instance.remove_prefix).to eq "abc"
+            expect(d.instance.add_prefix   ).to eq "123"
+            expect(d.instance.reject_keys  ).to eq "key1 key2"
+        end
+
+    end
+
+    describe 'Emit test ' do
+
+        it 'tag change(remove & add)' do
+            d = create_driver(%[
+                                   remove_prefix abc
+                                   add_prefix    123
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq '123.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq '123.def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(remove. & add)' do
+            d = create_driver(%[
+                                   remove_prefix abc
+                                   add_prefix    123.
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq '123.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq '123.def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(remove & add.)' do
+            d = create_driver(%[
+                                   remove_prefix abc
+                                   add_prefix    123.
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq '123.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq '123.def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(remove. & add.)' do
+            d = create_driver(%[
+                                   remove_prefix abc.
+                                   add_prefix    123.
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq '123.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq '123.def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(no_remove & add)' do
+            d = create_driver(%[
+                                   add_prefix    123
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq '123.abc.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq '123.abc.def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(remove & no_add)' do
+            d = create_driver(%[
+                                   remove_prefix abc
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq 'def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq 'def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(remove_not_match & add)' do
+            d = create_driver(%[
+                                   remove_prefix xyz
+                                   add_prefix    123
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq '123.abc.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq '123.abc.def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(not_remove(not perfect match) & add)' do
+            d = create_driver(%[
+                                   remove_prefix ab
+                                   add_prefix    123
+                                   reject_keys   key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq '123.abc.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq '123.abc.def.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change(all_remove & add)' do
+            d = create_driver(%[
+                                   remove_prefix old.tag
+                                   add_prefix    new.tag
+                                   reject_keys   key1
+                               ], 'old.tag')
+            d.run do
+                d.emit({'a' => 'b'},             time_now)
+                d.emit({'c' => 'd', 'e' => 'f'}, time_now)
+            end
+            emits = d.emits
+            expect(emits.size      ).to eq 2
+            expect(emits[0][0]     ).to eq 'new.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[1][0]     ).to eq 'new.tag'
+            expect(emits[1][2].size).to eq 2
+        end
+
+        it 'tag change & data reject1 & add)' do
+            d = create_driver(%[
+                                   remove_prefix abc
+                                   add_prefix    123
+                                   reject_keys   key
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'dat' => 'val'},                  time_now)
+                d.emit({'dat' => 'val', 'key' => 'vl'},   time_now)
+                d.emit({'dat' => 'val', 'key1' => 'vl1'}, time_now) 
+            end
+            emits = d.emits
+            rlt   = {'dat' => 'val'}
+            rlt1  = {'dat' => 'val', 'key1' => 'vl1'}
+            expect(emits.size      ).to eq 3
+            expect(emits[0][0]     ).to eq '123.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[0][2]     ).to eq rlt
+            expect(emits[1][0]     ).to eq '123.def.tag'
+            expect(emits[1][2].size).to eq 1
+            expect(emits[1][2]     ).to eq rlt
+            expect(emits[2][0]     ).to eq '123.def.tag'
+            expect(emits[2][2].size).to eq 2
+            expect(emits[2][2]     ).to eq rlt1
+        end
+
+        it 'tag change & data reject2 & add)' do
+            d = create_driver(%[
+                                   remove_prefix abc
+                                   add_prefix    123
+                                   reject_keys   key key1
+                               ], 'abc.def.tag')
+            d.run do
+                d.emit({'dat' => 'val'},                                                    time_now)
+                d.emit({'dat' => 'val', 'key'  => 'vl'},                                    time_now)
+                d.emit({'dat' => 'val', 'key1' => 'vl1'},                                   time_now)
+                d.emit({'dat' => 'val', 'key2' => 'vl2'},                                   time_now)
+                d.emit({'dat' => 'val', 'key'  => 'vl',  'key1' => 'vl1'},                  time_now)
+                d.emit({'dat' => 'val', 'key1' => 'vl1', 'key'  => 'vl'},                   time_now)
+                d.emit({'dat' => 'val', 'key'  => 'vl',  'key1' => 'vl1', 'key2' => 'vl2'}, time_now)
+                d.emit({'dat' => 'val', 'key'  => 'vl',  'key2' => 'vl2', 'key1' => 'vl1'}, time_now)
+                d.emit({'dat' => 'val', 'key1' => 'vl1', 'key2' => 'vl2', 'key'  => 'vl'},  time_now)
+            end
+            emits = d.emits
+            rlt   = {'dat' => 'val'}
+            rlt1  = {'dat' => 'val', 'key2' => 'vl2'}
+            expect(emits.size      ).to eq 9
+            expect(emits[0][0]     ).to eq '123.def.tag'
+            expect(emits[0][2].size).to eq 1
+            expect(emits[0][2]     ).to eq rlt
+            expect(emits[1][0]     ).to eq '123.def.tag'
+            expect(emits[1][2].size).to eq 1
+            expect(emits[1][2]     ).to eq rlt
+            expect(emits[2][0]     ).to eq '123.def.tag'
+            expect(emits[2][2].size).to eq 1
+            expect(emits[2][2]     ).to eq rlt
+            expect(emits[3][0]     ).to eq '123.def.tag'
+            expect(emits[3][2].size).to eq 2
+            expect(emits[3][2]     ).to eq rlt1
+            expect(emits[4][0]     ).to eq '123.def.tag'
+            expect(emits[4][2].size).to eq 1 
+            expect(emits[4][2]     ).to eq rlt
+            expect(emits[5][0]     ).to eq '123.def.tag'
+            expect(emits[5][2].size).to eq 1
+            expect(emits[5][2]     ).to eq rlt
+            expect(emits[6][0]     ).to eq '123.def.tag'
+            expect(emits[6][2].size).to eq 2
+            expect(emits[6][2]     ).to eq rlt1
+            expect(emits[7][0]     ).to eq '123.def.tag'
+            expect(emits[7][2].size).to eq 2
+            expect(emits[7][2]     ).to eq rlt1
+            expect(emits[8][0]     ).to eq '123.def.tag'
+            expect(emits[8][2].size).to eq 2
+            expect(emits[8][2]     ).to eq rlt1
+        end
+
+  end
+
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+# encoding: UTF-8
+require 'rubygems'
+require 'bundler'
+
+require 'rspec'
+require 'fluent/test'
+
+Dir["./lib/**/*.rb"].each{| f | require f}

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-data-rejecter
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Hirotaka Tajiri
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent plugin Output filer to reject key pair
+email: ganryu_koziro@excite.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-data-rejecter.gemspec
+- lib/fluent/plugin/out_data_rejecter.rb
+- spec/out_data_rejecter_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/hirotaka-tajiri/fluent-plugin-data-rejecter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Output filter plugin to reject key pair
+test_files:
+- spec/out_data_rejecter_spec.rb
+- spec/spec_helper.rb