fluent-plugin-record_splitter 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: faeed660e0eaba82e6fe96a667ccc4d4cc2eae39
+  data.tar.gz: 1d903b0a9fceb640c546ef5fc0dcf2ff30dbc2d3
+SHA512:
+  metadata.gz: c64f73f88e5fe20f7a74218ba45246e2a3b6f03aa075b143d4a23ef6023b91f658d47aeed3bc3d195b0fc8aac983f86fc520372874bffa3c40acb64ee56457b6
+  data.tar.gz: 3b217075b76f801f2db5ff8ab86ec54ff46361ce4fac8d13febe74f9766978b10fa3435cace3a5e104d0808a50743972a3fa980c5cfbe0ec9e38fde5f5f71059

data/.gitignore

@@ -0,0 +1,39 @@
+*.gem
+*.rbc
+*.swp
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+Gemfile.lock
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,69 @@
+fluent-plugin-record_splitter
+=====================
+
+Output split array plugin for fluentd.
+
+## Installation
+
+```
+gem install fluent-plugin-record_splitter
+```
+
+## Configuration
+
+    <match pattern>
+      type record_splitter
+      tag foo.splitted
+      split_key target_field
+      keep_keys ["common","general"]
+    </match>
+
+If following record is passed:
+
+```js
+{'common':'c', 'general':'g', 'other':'foo', 'target_field':[ {'k1':'v1'}, {'k2':'v2'} ] }
+```
+
+then you got new records like below:
+
+```js
+{'common':'c', 'general':'g', 'k1':'v1'}
+{'common':'c', 'general':'g', 'k2':'v2'}
+```
+
+another configuration
+
+    <match pattern>
+      type record_splitter
+      tag foo.splitted
+      split_key target_field
+      keep_other_key true
+      remove_keys ["general"]
+    </match>
+
+If following record is passed:
+
+```js
+{'common':'c', 'general':'g', 'other':'foo', 'target_field':[ {'k1':'v1'}, {'k2':'v2'} ] }
+```
+
+then you got new records like below:
+
+```js
+{'common':'c', 'other':'foo', 'k1':'v1'}
+{'common':'c', 'other':'foo', 'k2':'v2'}
+```
+
+## Copyright
+
+<table>
+  <tr>
+    <td>Author</td><td>Yuri Odagiri <ixixizko@gmail.com></td>
+  </tr>
+  <tr>
+    <td>Copyright</td><td>Copyright (c) 2015- Yuri Odagiri</td>
+  </tr>
+  <tr>
+    <td>License</td><td>MIT License</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler'
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.1.3

data/fluent-plugin-record_splitter.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-record_splitter"
+  gem.description = "output split array plugin for fluentd"
+  gem.homepage    = "https://github.com/ixixi/fluent-plugin-record_splitter"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Yuri Odagiri"]
+  gem.email       = "ixixizko@gmail.com"
+  gem.has_rdoc    = false
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
+  gem.add_dependency "fluent-mixin-rewrite-tag-name"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_record_splitter.rb

@@ -0,0 +1,60 @@
+require 'fluent/mixin/config_placeholders'
+require 'fluent/mixin/rewrite_tag_name'
+
+module Fluent
+  class RecordSplitterOutput < Output
+    Fluent::Plugin.register_output('record_splitter', self)
+
+    config_param :tag, :string
+    config_param :remove_prefix, :string, :default => nil
+    config_param :add_prefix, :string, :default => nil
+    config_param :split_key, :string
+    config_param :keep_other_key, :bool, :default => false
+    config_param :keep_keys, :array, :default => []
+    config_param :remove_keys, :array, :default => []
+
+    include SetTagKeyMixin
+    include Fluent::Mixin::ConfigPlaceholders
+    include Fluent::HandleTagNameMixin
+    include Fluent::Mixin::RewriteTagName
+
+    def configure(conf)
+      super
+      if not @keep_keys.empty? and not @remove_keys.empty?
+        raise Fluent::ConfigError, 'Cannot set both keep_keys and remove_keys.'
+      end
+      if @keep_other_key and not @keep_keys.empty?
+        raise Fluent::ConfigError, 'Cannot set keep_keys when keep_other_key is true.'
+      end
+      if not @keep_other_key and not @remove_keys.empty?
+        raise Fluent::ConfigError, 'Cannot set remove_keys when keep_other_key is false.'
+      end
+      if not @tag and not @remove_prefix and not @add_prefix
+        raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
+      end
+      if @tag and (@remove_prefix or @add_prefix)
+        raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
+      end
+    end
+
+    def emit(tag, es, chain)
+      emit_tag = tag.dup
+      es.each { |time, record|
+        filter_record(emit_tag, time, record)
+        if keep_other_key
+          common = record.reject{|key, value| key == @split_key or @remove_keys.include?(key) } 
+        else
+          common = record.select{|key, value| @keep_keys.include?(key) } 
+        end
+        if record.key?(@split_key)
+          record[@split_key].each{|v|
+            v.merge!(common) unless common.empty?
+            Engine.emit(emit_tag, time, v.merge(common))
+          }
+        end
+      }
+      chain.next
+    end
+
+  end
+end

data/test/test_out_record_splitter.rb

@@ -0,0 +1,92 @@
+require 'fluent/test'
+require 'fluent/plugin/out_record_splitter'
+
+
+class RecordSplitterOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type record_splitter
+    tag foo.splitted
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::RecordSplitterOutput, tag='test_tag').configure(conf)
+  end
+
+  def test_split_default
+    d = create_driver %[
+      type record_splitter
+      tag foo.splitted
+      split_key target_field
+    ]
+
+    d.run do
+      d.emit('other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    end
+
+    assert_equal [
+      {'k1'=>'v1'},
+      {'k2'=>'v2'},
+    ], d.records
+  end
+
+
+  def test_split_with_keep_other_key
+    d = create_driver %[
+      type record_splitter
+      tag foo.splitted
+      split_key target_field
+      keep_other_key true
+    ]
+
+    d.run do
+      d.emit('common'=>'c','general'=>'g','other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    end
+
+    assert_equal [
+      {'common'=>'c','general'=>'g','other'=>'foo','k1'=>'v1'},
+      {'common'=>'c','general'=>'g','other'=>'foo','k2'=>'v2'},
+    ], d.records
+  end
+
+  def test_split_with_keep_other_key_and_remove_key
+    d = create_driver %[
+      type record_splitter
+      tag foo.splitted
+      split_key target_field
+      keep_other_key true
+      remove_keys ["general","other"]
+    ]
+
+    d.run do
+      d.emit('common'=>'c','general'=>'g','other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    end
+
+    assert_equal [
+      {'common'=>'c','k1'=>'v1'},
+      {'common'=>'c','k2'=>'v2'},
+    ], d.records
+  end
+
+  def test_split_with_keep_keys
+    d = create_driver %[
+      type record_splitter
+      tag foo.splitted
+      split_key target_field
+      keep_keys ["common","general"]
+    ]
+
+    d.run do
+      d.emit('common'=>'c','general'=>'g','other'=>'foo','target_field' => [{'k1'=>'v1'},{'k2'=>'v2'}])
+    end
+
+    assert_equal [
+      {'common'=>'c','general'=>'g','k1'=>'v1'},
+      {'common'=>'c','general'=>'g','k2'=>'v2'},
+    ], d.records
+  end
+
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-record_splitter
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Yuri Odagiri
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-rewrite-tag-name
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: output split array plugin for fluentd
+email: ixixizko@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-record_splitter.gemspec
+- lib/fluent/plugin/out_record_splitter.rb
+- test/test_out_record_splitter.rb
+homepage: https://github.com/ixixi/fluent-plugin-record_splitter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: output split array plugin for fluentd
+test_files:
+- test/test_out_record_splitter.rb