fluent-plugin-record-serializer 0.1.0

data/.gitignore

@@ -0,0 +1,14 @@
+.bundle
+.yardoc
+Gemfile.lock
+_yardoc
+coverage
+doc
+pkg
+spec/reports
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - 2.2.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-record-serializer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2015- IKUTA Masahito
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,56 @@
+# fluent-plugin-record-serializer
+
+[fluentd](http://fluentd.org) filter plugin that serialize a record.
+
+[![Build Status](https://travis-ci.org/cooldaemon/fluent-plugin-record-serializer.svg?branch=master)](https://travis-ci.org/cooldaemon/fluent-plugin-record-serializer)
+[![Code Climate](https://codeclimate.com/github/cooldaemon/fluent-plugin-record-serializer/badges/gpa.svg)](https://codeclimate.com/github/cooldaemon/fluent-plugin-record-serializer)
+
+If following record is passed:
+
+```
+{"message": "hello world!"}
+```
+
+then you got new record like below:
+
+```
+{"tag": "pattern", "payload": "{\"message\": \"hello world!\"}"}
+```
+
+## Installation
+
+Install with gem or fluent-gem command as:
+
+```
+# for fluentd
+$ gem install fluent-plugin-record-serializer
+
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-record-serializer
+```
+
+## Configuration
+
+In v0.12, you can use record_modifier filter.
+
+```
+<filter pattern>
+  type record_serializer
+</filter>
+```
+
+In v0.10, you can use record_serializer output to emulate filter.
+
+```
+<match pattern>
+  type record_serializer
+  tag serialized.pattern
+</match>
+```
+
+## Copyright
+
+- Copyright
+  - Copyright(C) 2015- IKUTA Masahito (cooldaemon)
+- License
+  - Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-record-serializer.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/record/serializer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-record-serializer"
+  spec.version       = Fluent::Plugin::Record::Serializer::VERSION
+  spec.authors       = ["IKUTA Masahito"]
+  spec.email         = ["cooldaemon@gmail.com"]
+  spec.summary       = %q{Fluentd output filter plugin for serialize record.}
+  spec.description   = spec.summary
+  spec.homepage      = "http://github.com/cooldaemon/fluent-plugin-record-serializer"
+  spec.license       = "APLv2"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", ["~> 3.0", "~> 3.1"]
+
+  spec.add_dependency "fluentd", ">= 0.10.53"
+end

data/lib/fluent/plugin/filter_record_serializer.rb

@@ -0,0 +1,24 @@
+require 'fluent/plugin/record_serializer'
+
+module Fluent
+  class RecordSerializerFilter < Filter
+    Fluent::Plugin.register_filter('record_serializer', self)
+
+    config_param :format, :string, :default => 'json'
+    config_param :field_name, :string, :default => 'payload'
+
+    include SetTagKeyMixin
+    include Fluent::RecordSerializer
+
+    def filter(tag, time, record)
+      begin
+        serialized_record = serialize_record(@format, record)
+      rescue => e
+        $log.warn "serialize error: #{e.message}"
+        return
+      end
+
+      {'tag' => tag, @field_name => serialized_record}
+    end
+  end
+end

data/lib/fluent/plugin/out_record_serializer.rb

@@ -0,0 +1,36 @@
+require 'fluent/plugin/record_serializer'
+
+module Fluent
+  class RecordSerializerOutput < Output
+    Fluent::Plugin.register_output('record_serializer', self)
+
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end 
+
+    config_param :tag, :string
+    config_param :format, :string, :default => 'json'
+    config_param :field_name, :string, :default => 'payload'
+
+    include SetTagKeyMixin
+    include Fluent::RecordSerializer
+
+    def emit(tag, es, chain)
+      es.each { |time, record|
+        begin
+          serialized_record = serialize_record(@format, record)
+        rescue => e
+          $log.warn "serialize error: #{e.message}"
+          next
+        end
+
+        router.emit(@tag, time, {
+          'tag' => @tag,
+          @field_name => serialized_record
+        })
+      }
+
+      chain.next
+    end
+  end
+end

data/lib/fluent/plugin/record/serializer/version.rb

@@ -0,0 +1,9 @@
+module Fluent
+  module Plugin
+    module Record
+      module Serializer
+        VERSION = '0.1.0'
+      end
+    end
+  end
+end

data/lib/fluent/plugin/record_serializer.rb

@@ -0,0 +1,24 @@
+module Fluent
+  module RecordSerializer
+    def serialize_record(format, record)
+      if format == 'json'
+        json_dump(record)
+      elsif format == 'yaml'
+        record.to_yaml
+      end
+    end
+
+    private
+
+    begin
+      require 'oj'
+      def json_dump(record)
+        Oj.dump(record, :mode => :compat)
+      end
+    rescue LoadError
+      def json_dump(record)
+        record.to_json
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fileutils'
+require 'fluent/log'
+require 'fluent/test'
+
+$log = Fluent::Log.new(STDOUT, Fluent::Log::LEVEL_DEBUG)
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_filter_record_serializer.rb

@@ -0,0 +1,56 @@
+require 'helper'
+require 'fluent/plugin/filter_record_serializer'
+require 'yaml'
+
+exit unless defined?(Fluent::Filter)
+
+class RecordSerializerFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  TAG = 'test.serialize'
+  FORMAT = 'json'
+  FIELD_NAME = 'payload'
+
+  CONFIG = %[
+    type record_modifier
+  ]
+
+  YAML_CONFIG = %[
+    type record_modifier
+    format yaml
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::FilterTestDriver.new(Fluent::RecordSerializerFilter, TAG).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal FORMAT, d.instance.format
+    assert_equal FIELD_NAME, d.instance.field_name
+  end
+
+  def test_json
+    record = {'spam' => 1, 'ham' => 'egg',}
+    d = create_driver
+
+    d.run do
+      d.emit(record)
+    end
+
+    assert_equal [{'tag' => TAG, FIELD_NAME => record.to_json}], d.filtered_as_array.map { |e| e.last }
+  end
+
+  def test_yaml
+    record = {'spam' => 1, 'ham' => 'egg',}
+    d = create_driver YAML_CONFIG
+
+    d.run do
+      d.emit(record)
+    end
+
+    assert_equal [{'tag' => TAG, FIELD_NAME => record.to_yaml}], d.filtered_as_array.map { |e| e.last }
+  end
+end

data/test/plugin/test_out_record_serializer.rb

@@ -0,0 +1,54 @@
+require 'helper'
+require 'fluent/plugin/out_record_serializer'
+require 'yaml'
+
+class RecordSerializerOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  TAG = 'test.serialize'
+  FORMAT = 'json'
+  FIELD_NAME = 'payload'
+
+  CONFIG = %[
+    tag #{TAG}
+  ]
+
+  YAML_CONFIG = %[
+    tag #{TAG}
+    format yaml
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::RecordSerializerOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal FORMAT, d.instance.format
+    assert_equal FIELD_NAME, d.instance.field_name
+  end
+
+  def test_json
+    record = {'spam' => 1, 'ham' => 'egg',}
+    d = create_driver
+
+    d.run do
+      d.emit(record)
+    end
+
+    assert_equal [{'tag' => TAG, FIELD_NAME => record.to_json}], d.records
+  end
+
+  def test_yaml
+    record = {'spam' => 1, 'ham' => 'egg',}
+    d = create_driver YAML_CONFIG
+
+    d.run do
+      d.emit(record)
+    end
+
+    assert_equal [{'tag' => TAG, FIELD_NAME => record.to_yaml}], d.records
+  end
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification 
+name: fluent-plugin-record-serializer
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- IKUTA Masahito
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2015-11-29 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  type: :development
+  name: rake
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 10
+        - 0
+        version: "10.0"
+  requirement: *id001
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  type: :development
+  name: test-unit
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 3
+        - 0
+        version: "3.0"
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 3
+        - 1
+        version: "3.1"
+  requirement: *id002
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  type: :runtime
+  name: fluentd
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 10
+        - 53
+        version: 0.10.53
+  requirement: *id003
+description: Fluentd output filter plugin for serialize record.
+email: 
+- cooldaemon@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-record-serializer.gemspec
+- lib/fluent/plugin/filter_record_serializer.rb
+- lib/fluent/plugin/out_record_serializer.rb
+- lib/fluent/plugin/record/serializer/version.rb
+- lib/fluent/plugin/record_serializer.rb
+- test/helper.rb
+- test/plugin/test_filter_record_serializer.rb
+- test/plugin/test_out_record_serializer.rb
+has_rdoc: true
+homepage: http://github.com/cooldaemon/fluent-plugin-record-serializer
+licenses: 
+- APLv2
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Fluentd output filter plugin for serialize record.
+test_files: 
+- test/helper.rb
+- test/plugin/test_filter_record_serializer.rb
+- test/plugin/test_out_record_serializer.rb