fluent-plugin-key-picker 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d7d01f21018a0789fb1216c54cbaa32b3338b53d
+  data.tar.gz: 751d86fe1d11bca30250763d83c357c07c30a51b
+SHA512:
+  metadata.gz: 5dd5dd150f144a1e724abbe05ddc513ae4ff837b691f408c8244fe4ee0bca3bbe874183a2e437d931562bca949168cefc37eac369204848f8bd69c0ade74e42e
+  data.tar.gz: 4b70cf884c3251eeb723d12f6d49eb20e297039602a795ba4d91ee3500860e1ea10407386548588d75a26e9167eeee1353066b0440688d14611c7bfd55ec000d

data/.gitignore

@@ -0,0 +1,4 @@
+.bundle
+*.swp
+.DS_Store
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,2 @@
+Copyright (c) 2014 Carlos Donderis
+Apache License, Version 2.0

data/README.md

@@ -0,0 +1,85 @@
+# fluent-plugin-time_parser, a plugin for [Fluentd](http://fluentd.org)
+
+## Component
+KeyPickerOutput
+
+Take a record and pick only the keys you want to pass
+
+## wat?!
+
+```
+<match test.**>
+  type key_picker
+
+  keys           foo, baz
+  add_tag_prefix picked.
+</match>
+```
+
+And you feed such a value into fluentd:
+
+```
+"test" => {
+  "foo" => "a",
+  "bar" => "b",
+  "baz" => "c",
+  "bah" => "e",
+}
+```
+
+Then you'll get re-emmited tags/records like so:
+
+```
+"picked.test" => {
+  "foo" => "a",
+  "baz" => "c"
+}
+```
+### WARNING
+  Unmatched keys will DISAPPEAR, please use this with caution.
+
+## Configuration
+
+### keys
+
+`keys` should include the keys you want to pick.
+
+### remove_tag_prefix, remove_tag_suffix, add_tag_prefix, add_tag_suffix
+
+These params are included from `Fluent::HandleTagNameMixin`. See the code for details.
+
+You must add at least one of these params.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-key-picker'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-key-picker
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright
+
+### Copyright
+
+Copyright (c) 2014- Carlos Donderis (@CaDs)
+
+### License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-key-picker.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-key-picker'
+  gem.version       = '0.0.1'
+  gem.authors       = ['Carlos Donderis']
+  gem.email         = ['cdonderis@gmail.com']
+  gem.homepage      = 'http://github.com/cads/fluent-plugin-key-picker'
+  gem.description   = %q{Fluentd plugin for filtering / picking desired keys.}
+  gem.summary       = %q{This plugin allows you specify only keys you want to use.}
+  gem.license       = 'Apache License, Version 2.0'
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'rake', '~> 0'
+  gem.add_development_dependency 'fluentd', '~> 0'
+end

data/lib/fluent/plugin/out_key_picker.rb

@@ -0,0 +1,57 @@
+# coding: utf-8
+
+module Fluent
+  class KeyPickerOutput < Output
+    include Fluent::HandleTagNameMixin
+
+    Fluent::Plugin.register_output('key_picker', self)
+
+    config_param :keys, :string, :default => nil
+    config_param :add_tag_and_reemit, :bool, :default => false
+
+    def configure(conf)
+      super
+
+      if (
+          !remove_tag_prefix &&
+          !remove_tag_suffix &&
+          !add_tag_prefix    &&
+          !add_tag_suffix
+      )
+        raise ConfigError, "out_key_picker: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
+      end
+
+      unless keys && !keys.nil?
+        raise ConfigError, "keys: You need to specify the keys you want to pick."
+      end
+
+      @keys = keys && keys.split(/\s*,\s*/)
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each {|time,record|
+        t = tag.dup
+        filter_record(t, time, record)
+        Engine.emit(t, time, record)
+      }
+      chain.next
+    end
+
+    def filter_record(tag, time, record)
+      begin
+        record.keep_if{|key, value| @keys.include?(key.to_s)}
+      rescue ArgumentError => error
+        $log.warn("out_key_picker: #{error.message}")
+      end
+      super(tag, time, record)
+    end
+  end
+end

data/test/plugin/test_out_key_picker.rb

@@ -0,0 +1,118 @@
+require 'byebug'
+# -*- encoding: utf-8 -*-
+require '../test_helper'
+
+class KeyPickerOutputTest < Test::Unit::TestCase
+
+  TEST_RECORD = {'foo' => 'a', 'bar' => 'b', 'baz' => 'c'}
+  EMPTY_HASH = {}
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(
+      Fluent::KeyPickerOutput, tag
+    ).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      keys           foo, bar
+      add_tag_prefix picked.
+    ])
+    assert_equal ['foo', 'bar'], d.instance.keys
+    assert_equal 'picked.', d.instance.add_tag_prefix
+
+    #No Prefix
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        keys      foo, bar
+      ])
+    end
+
+    #No Keys
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        add_tag_prefix picked.
+      ])
+    end
+  end
+
+  def test_filter_record
+    d = create_driver(%[
+      keys           foo, bar
+      add_tag_prefix picked.
+    ])
+
+    record = TEST_RECORD.dup
+    d.instance.filter_record('test', Time.now, record)
+    assert_equal record, {'foo' => 'a', 'bar' => 'b'}
+    assert_equal record['foo'], 'a'
+    assert_equal record['bar'], 'b'
+  end
+
+
+  def test_no_matching_record
+    d = create_driver(%[
+      keys           foo, bar
+      add_tag_prefix picked.
+    ])
+
+    tag    = 'test'
+    record = {}
+
+    assert_equal record, {}
+  end
+
+  def test_emit
+    d = create_driver(%[
+      keys           foo, bar
+      add_tag_prefix picked.
+    ])
+
+    d.run { d.emit(TEST_RECORD.dup) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'picked.test', emits[0][0]
+    assert_equal 'a', emits[0][2]['foo']
+    assert_equal 'b', emits[0][2]['bar']
+  end
+
+  def test_emit_multi
+    d = create_driver(%[
+      keys           foo, bar
+      add_tag_prefix picked.
+    ])
+    d.run do
+      d.emit(TEST_RECORD.dup)
+      d.emit(TEST_RECORD.dup)
+      d.emit(TEST_RECORD.dup)
+    end
+    emits = d.emits
+
+    assert_equal 3, emits.count
+
+    emits.each do |e|
+      assert_equal 'picked.test', e[0]
+      assert_equal 'a', e[2]['foo']
+      assert_equal 'b', e[2]['bar']
+    end
+  end
+
+  def test_emit_without_match_key
+    d = create_driver(%[
+      keys           fiz, faz
+      add_tag_prefix picked.
+    ])
+    d.run { d.emit(TEST_RECORD.dup) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'picked.test', emits[0][0]
+    assert_equal EMPTY_HASH, emits[0][2]
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_key_picker'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-key-picker
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Carlos Donderis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin for filtering / picking desired keys.
+email:
+- cdonderis@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-key-picker.gemspec
+- lib/fluent/plugin/out_key_picker.rb
+- test/plugin/test_out_key_picker.rb
+- test/test_helper.rb
+homepage: http://github.com/cads/fluent-plugin-key-picker
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: This plugin allows you specify only keys you want to use.
+test_files:
+- test/plugin/test_out_key_picker.rb
+- test/test_helper.rb