fluent-mixin-type-converter 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+vendor/*

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+
+rvm:
+  - 2.1.0
+  - 2.0.0
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fluent-mixin-type-converter.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2014- Kentaro Yoshida
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/README.md

@@ -0,0 +1,164 @@
+## Fluent::Mixin::TypeConverter [![Build Status](https://travis-ci.org/y-ken/fluent-mixin-type-converter.png?branch=master)](https://travis-ci.org/y-ken/fluent-mixin-type-converter)
+
+## Overview
+
+Fluentd mixin plugin to provides type conversion function as like as in_tail plugin. It acts calling [Fluent::TextParser::TypeConverter](https://github.com/fluent/fluentd/blob/master/lib/fluent/parser.rb) as mixin. It will let you get easy to implement type conversion for your own plugins.
+
+## Function
+
+It supportes these type conversion.
+
+- string
+- integer
+- float
+- bool
+- time
+- array
+
+#### Option
+
+* `types_delimiter`  
+[default] `,`
+
+* `types_label_delimiter`  
+[default] `:`
+
+## Configuration
+
+Adding this mixin plugin, it will enabled to use these type conversion in your plugins.
+
+```xml
+# input plugin example
+<source>
+  type   foo_bar
+  tag    test.message
+  
+  # type conversion with this rule before emit.
+  types  member_id:int,temperature:float
+</source>
+```
+
+```xml
+# output plugin example
+<match test.foo>
+  type   foo_bar
+  tag    test.message
+
+  # type conversion with this rule before emit.
+  types  member_id:int,temperature:float
+</match>
+```
+
+Another examples are written in [unit test](https://github.com/y-ken/fluent-mixin-type-converter/blob/master/test/mixin/test_type_converter).
+
+## Usage
+
+#### 1. edit gemspec
+
+add dependency for .gemspec file like below. For more detail, see [gemspec example](https://github.com/y-ken/fluent-plugin-watch-process/blob/master/fluent-plugin-watch-process.gemspec)
+
+```ruby
+spec.add_runtime_dependency "fluent-mixin-type-converter"
+```
+
+#### 2. activate fluent-mixin-type-converter for your plugin
+
+It is the instruction in the case of adding `fluent-plugin-foobar`.
+
+```
+$ cd fluent-plugin-foobar
+$ vim fluent-plugin-foobar.gemspec # edit gemspec
+$ bundle install --path vendor/bundle # or just type `bundle install`
+```
+
+#### 3. edit your plugin to implement
+
+It is a quick guide to enable your plugin to use TypeConverter Mixin.  
+The key points of basic implmentation is just like below.
+
+* add `require 'fluent/mixin/type_converter'` at the top of source
+* in the case of output plugin, add `include Fluent::HandleTagNameMixin` (recommend)  
+this is required if you will use kind of 'remove_tag_prefix' option together
+* add `include Fluent::Mixin::TypeConverter` in intput/output class after HandleTagNameMixin
+* add `emit_tag = tag.dup` and `filter_record(emit_tag, time, record)` before `Engine.emit`
+
+##### implement example for input plugin
+
+```ruby
+require 'fluent/mixin/type_converter
+
+module Fluent
+  class FooBarInput < Fluent::Input
+    Plugin.register_input('foo_bar', self)
+
+    # ...snip...
+    
+    include Fluent::Mixin::TypeConverter
+    config_param :types, :string, :default => nil
+    config_param :types_delimiter, :string, :default => ','
+    config_param :types_label_delimiter, :string, :default => ':'
+
+    # ...snip...
+
+    def emit_message(tag, message)
+      emit_tag = tag.dup
+      filter_record(emit_tag, Engine.now, message)
+      Engine.emit(emit_tag, Engine.now, message)
+    end
+
+    # ...snip...
+
+  end
+end
+```
+
+##### implement example for output plugin
+
+```ruby
+require 'fluent/mixin/type_converter
+
+class Fluent
+  class FooBarOutput < Fluent::Output
+    Fluent::Plugin.register_output('foo_bar', self)
+
+    # ...snip...
+
+    include Fluent::Mixin::TypeConverter
+    config_param :types, :string, :default => nil
+    config_param :types_delimiter, :string, :default => ','
+    config_param :types_label_delimiter, :string, :default => ':'
+
+    # ...snip...
+
+    def emit(tag, es, chain)
+      es.each do |time, record|
+        emit_tag = tag.dup
+        filter_record(emit_tag, time, record)
+        Fluent::Engine.emit(emit_tag, time, record)
+      end
+      chain.next
+    end
+
+    # ...snip...
+
+  end
+end
+```
+
+## Case Study
+
+These cool plugins are using this mixin!
+
+* [fluent-plugin-watch-process](https://github.com/y-ken/fluent-plugin-watch-process/)
+
+## TODO
+
+Pull requests are very welcome!!
+
+## Copyright
+
+Copyright © 2014- Kentaro Yoshida ([@yoshi_ken](https://twitter.com/yoshi_ken))
+
+## License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-mixin-type-converter.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-mixin-type-converter"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Kentaro Yoshida"]
+  spec.email         = ["y.ken.studio@gmail.com"]
+  spec.summary       = %q{Fluentd mixin plugin to provides type conversion function as like as in_tail plugin. It acts calling Fluent::TextParser::TypeConverter as mixin. It will let you get easy to implement type conversion for your own plugins.}
+  spec.homepage      = "https://github.com/y-ken/fluent-mixin-type-converter"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "fluent-mixin-rewrite-tag-name"
+  spec.add_runtime_dependency "fluentd"
+end

data/lib/fluent/mixin/type_converter.rb

@@ -0,0 +1,31 @@
+module Fluent
+  module Mixin
+    module TypeConverter
+      include Configurable
+      include RecordFilterMixin
+      include Fluent::TextParser::TypeConverter
+
+      attr_accessor :types, :types_delimiter, :types_label_delimiter
+
+      def configure(conf)
+        super
+
+      end
+
+      def filter_record(tag, time, record)
+        super
+        if @types
+          convert_field_type!(record)
+        end
+      end
+
+      def convert_field_type!(record)
+        record.each { |key, value|
+          record[key] = convert_type(key, value)
+        }
+        self
+      end
+
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/mixin/type_converter'
+require_relative 'plugin'
+
+class Test::Unit::TestCase
+end

data/test/mixin/test_type_converter.rb

@@ -0,0 +1,93 @@
+require 'helper'
+
+class TypeConverterMixinTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    tag                log.${tag}
+    remove_tag_prefix  apache.
+    types  code:integer,response_time:float
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::TypeConverterMixinOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('unknown_keys')
+    }
+    d = create_driver(CONFIG)
+    puts d.instance.inspect
+    assert_equal 'code:integer,response_time:float', d.instance.config['types']
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG, 'apache.access')
+    d1.run do
+      d1.emit({
+        'code' => '200',
+        'response_time' => '0.03'
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'log.access', emits[0][0] # tag
+    assert_equal 200, emits[0][2]['code']
+    assert_equal 0.03, emits[0][2]['response_time']
+  end
+
+  def test_emit_with_custom_delimiter
+    d1 = create_driver(%[
+      tag    test.message
+      types  code:integer|response_time:float
+      types_delimiter       |
+      types_label_delimiter :
+    ], 'apache.access')
+    d1.run do
+      d1.emit({
+        'code' => '200',
+        'response_time' => '0.03'
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'test.message', emits[0][0] # tag
+    assert_equal 200, emits[0][2]['code']
+    assert_equal 0.03, emits[0][2]['response_time']
+  end
+
+  def test_emit_all_types
+    d1 = create_driver(%[
+      tag    test.message
+      types  string:string,integer:integer,float:float,bool:bool,time:time,array:array
+    ], 'apache.access')
+    d1.run do
+      d1.emit({
+        'string'  => 'foo',
+        'integer' => '500',
+        'float'   => '0.5',
+        'bool'    => 'true',
+        'time'    => '2014-01-21 12:29:44 +0900',
+        'array'   => 'foo,1,2.0,true'
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'test.message', emits[0][0] # tag
+    assert_equal 'foo', emits[0][2]['string']
+    assert_equal 500, emits[0][2]['integer']
+    assert_equal 0.5, emits[0][2]['float']
+    assert_equal true, emits[0][2]['bool']
+    assert_equal 1390274984, emits[0][2]['time']
+    assert_equal ["foo", "1", "2.0", "true"], emits[0][2]['array']
+  end
+end

data/test/plugin.rb

@@ -0,0 +1,31 @@
+require 'fluent/mixin/rewrite_tag_name'
+
+class Fluent::TypeConverterMixinOutput < Fluent::Output
+  Fluent::Plugin.register_output('type_converter_mixin', self)
+
+  config_param :tag, :string, :default => nil
+  config_param :types, :string, :default => nil
+  config_param :types_delimiter, :string, :default => ','
+  config_param :types_label_delimiter, :string, :default => ':'
+
+  include Fluent::HandleTagNameMixin
+  include Fluent::Mixin::RewriteTagName
+  include Fluent::Mixin::TypeConverter
+
+  def configure(conf)
+    super
+
+    if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
+      raise Fluent::ConfigError, "RewriteTagNameMixin: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
+    end
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      emit_tag = tag.dup
+      filter_record(emit_tag, time, record)
+      Fluent::Engine.emit(emit_tag, time, record)
+    end
+    chain.next
+  end
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: fluent-mixin-type-converter
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Kentaro Yoshida
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-rewrite-tag-name
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- y.ken.studio@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-mixin-type-converter.gemspec
+- lib/fluent/mixin/type_converter.rb
+- test/helper.rb
+- test/mixin/test_type_converter.rb
+- test/plugin.rb
+homepage: https://github.com/y-ken/fluent-mixin-type-converter
+licenses:
+- Apache License, Version 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluentd mixin plugin to provides type conversion function as like as in_tail
+  plugin. It acts calling Fluent::TextParser::TypeConverter as mixin. It will let
+  you get easy to implement type conversion for your own plugins.
+test_files:
+- test/helper.rb
+- test/mixin/test_type_converter.rb
+- test/plugin.rb