fluent-mixin-config-placeholders 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-mixin-config-placeholders.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2012- TAGOMORI Satoshi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,42 @@
+# Fluent::Mixin::ConfigPlaceholders
+
+Fluent::Mixin::ConfigPlaceHolders provide some placeholders to fluentd plugins that includes this mix-in. Placeholders below are expanded in 'super' of including plugin's #configure method.
+
+Available placeholders are:
+
+* hostname (${hostname} or \_\_HOSTNAME\_\_)
+  * you can specify hostname string explicitly on 'hostname' parameter
+* random uuid (${uuid}, ${uuid:random}, \_\_UUID\_\_ or \_\_UUID\_RANDOM\_\_)
+* hostname string based uuid (${uuid:hostname} or \_\_UUID\_HOSTNAME\_\_)
+* timestamp based uuid (${uuid:timestamp} or \_\_UUID\_TIMESTAMP\_\_)
+
+## Usage
+
+In plugin (both of input and output), just include mixin.
+
+    class FooInput < Fluent::Input
+      Fluent::Plugin.register_input('foo', self)
+    
+      config_param :tag, :string
+      
+      include Fluent::Mixin::ConfigPlaceholders
+    
+      def configure(conf)
+        super # MUST call 'super' at first!
+        
+        @tag #=> here, you can get string replaced '${hostname}' into actual hostname
+      end
+      
+      # ...
+    end
+
+You can use this feature for tags for each fluentd node, paths for remote storage services like /root/${hostname}/access_log or non-race-condition paths like /files/${uuid:random}.
+
+## AUTHORS
+
+* TAGOMORI Satoshi <tagomoris@gmail.com>
+
+## LICENSE
+
+* Copyright: Copyright (c) 2012- tagomoris
+* License: Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-mixin-config-placeholders.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-mixin-config-placeholders"
+  gem.version       = "0.1.0"
+  gem.authors       = ["TAGOMORI Satoshi"]
+  gem.email         = ["tagomoris@gmail.com"]
+  gem.description   = %q{to add various placeholders for plugin configurations}
+  gem.summary       = %q{Configuration syntax extension mixin for fluentd plugin}
+  gem.homepage      = "https://github.com/tagomoris/fluent-mixin-config-placeholders"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "fluentd"
+  gem.add_development_dependency "uuidtools"
+  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "uuidtools"
+end

data/lib/fluent/mixin/config_placeholders.rb

@@ -0,0 +1,74 @@
+require 'fluent/config'
+require 'uuidtools'
+
+module Fluent
+  module Mixin
+    module ConfigPlaceholders
+      attr_accessor :hostname
+
+      # ${hostname}, __HOSTNAME__
+
+      # ${uuid} or ${uuid:random} , __UUID__ or __UUID_RANDOM__
+      # UUIDTools::UUID.random_create
+      # => #<UUID:0x19013a UUID:984265dc-4200-4f02-ae70-fe4f48964159>
+
+      # ${uuid:hostname} , __UUID_HOSTNAME__
+      # UUIDTools::UUID.sha1_create(UUIDTools::UUID_DNS_NAMESPACE, "www.widgets.com")
+      # => #<UUID:0x2a0116 UUID:21f7f8de-8051-5b89-8680-0195ef798b6a>
+
+      # ${uuid:timestamp} , __UUID_TIMESTAMP__
+      # UUIDTools::UUID.timestamp_create
+      # => #<UUID:0x2adfdc UUID:64a5189c-25b3-11da-a97b-00c04fd430c8>
+
+      def uuid_random
+        UUIDTools::UUID.random_create.to_s
+      end
+
+      def uuid_hostname(hostname)
+        UUIDTools::UUID.sha1_create(UUIDTools::UUID_DNS_NAMESPACE, hostname).to_s
+      end
+
+      def uuid_timestamp
+        UUIDTools::UUID.timestamp_create.to_s
+      end
+
+      def replace(map, value)
+        map.reduce(value){|r,p| r.gsub(p[0], p[1].call())}
+      end
+
+      def configure(conf)
+        # Element#has_key? inserts key name into 'used' list, so we should escape that method...
+        hostname = conf.keys.include?('hostname') ? conf['hostname'] : `hostname`.chomp
+
+        mapping = {
+          '${hostname}'  => lambda{ hostname },
+          '__HOSTNAME__' => lambda{ hostname },
+          '${uuid}'         => lambda{ uuid_random() },
+          '__UUID__'        => lambda{ uuid_random() },
+          '${uuid:random}'  => lambda{ uuid_random() },
+          '__UUID_RANDOM__' => lambda{ uuid_random() },
+          '${uuid:hostname}'  => lambda { uuid_hostname(hostname) },
+          '__UUID_HOSTNAME__' => lambda { uuid_hostname(hostname) },
+          '${uuid:timestamp}'  => lambda { uuid_timestamp() },
+          '__UUID_TIMESTAMP__' => lambda { uuid_timestamp() },
+        }
+
+        def check_element(map,c)
+          c.arg = replace(map, c.arg)
+          c.keys.each do |k|
+            v = c.fetch(k, nil)
+            if v and v.is_a? String
+              c[k] = replace(map, v)
+            end
+          end
+          c.elements.each{|e| check_element(map,e)}
+        end
+
+        check_element(mapping,conf)
+
+        super
+      end
+
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,30 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/mixin/config_placeholders'
+require_relative 'plugin'
+
+class Test::Unit::TestCase
+end

data/test/mixin/test_config_placeholders.rb

@@ -0,0 +1,159 @@
+require 'helper'
+
+class ConfigPlaceholdersTest < Test::Unit::TestCase
+  def create_plugin_instances(conf)
+    [
+      Fluent::ConfigPlaceholdersTest0Input, Fluent::ConfigPlaceholdersTest1Input, Fluent::ConfigPlaceholdersTest2Input
+    ].map{|klass| Fluent::Test::InputTestDriver.new(klass).configure(conf).instance }
+  end
+
+  def test_unused
+    conf = %[
+tag HOGE
+path POSPOSPOS
+]
+    p = Fluent::Test::InputTestDriver.new(Fluent::ConfigPlaceholdersTest2Input).configure(conf).instance
+    assert_equal ['tag','path'], p.conf.used
+
+    conf = %[
+tag HOGE
+path POSPOSPOS
+]
+    p = Fluent::Test::InputTestDriver.new(Fluent::ConfigPlaceholdersTestXInput).configure(conf).instance
+    assert_equal [], p.conf.used
+
+    conf = %[
+tag HOGE
+path POSPOSPOS ${hostname} MOGEMOGE
+]
+    p = Fluent::Test::InputTestDriver.new(Fluent::ConfigPlaceholdersTestXInput).configure(conf).instance
+    assert_equal [], p.conf.used
+  end
+
+  def test_hostname
+    conf = %[
+hostname testing.local
+tag out.${hostname}
+path /path/to/file.__HOSTNAME__.txt
+]
+    p1, p2, p3 = create_plugin_instances(conf)
+
+    assert_equal 'out.${hostname}', p1.tag
+    assert_equal 'out.testing.local', p2.tag
+    assert_equal 'out.testing.local', p3.tag
+
+    assert_equal '/path/to/file.__HOSTNAME__.txt', p1.path
+    assert_equal '/path/to/file.testing.local.txt', p2.path
+    assert_equal '/PATH/TO/FILE.TESTING.LOCAL.TXT', p3.path
+  end
+
+  PATH_CHECK_REGEXP = Regexp.compile('^/path/to/file\.[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}.txt$')
+  PATH_CHECK_REGEXP2 = Regexp.compile('^/PATH/TO/FILE\.[0-9A-F]{8}-([0-9A-F]{4}-){3}[0-9A-F]{12}.TXT$')
+
+  def test_uuid_random
+    conf1 = %[
+tag test.out
+path /path/to/file.${uuid}.txt
+]
+    p1, p2, p3 = create_plugin_instances(conf1)
+    assert_match PATH_CHECK_REGEXP, p2.path
+    assert_match PATH_CHECK_REGEXP2, p3.path
+
+    conf2 = %[
+tag test.out
+path /path/to/file.${uuid:random}.txt
+]
+    p1, p2, p3 = create_plugin_instances(conf2)
+    assert_match PATH_CHECK_REGEXP, p2.path
+    assert_match PATH_CHECK_REGEXP2, p3.path
+
+    conf3 = %[
+tag test.out
+path /path/to/file.__UUID__.txt
+]
+    p1, p2, p3 = create_plugin_instances(conf3)
+    assert_match PATH_CHECK_REGEXP, p2.path
+    assert_match PATH_CHECK_REGEXP2, p3.path
+
+    conf4 = %[
+tag test.out
+path /path/to/file.__UUID__.txt
+]
+    p1, p2, p3 = create_plugin_instances(conf4)
+    assert_match PATH_CHECK_REGEXP, p2.path
+    assert_match PATH_CHECK_REGEXP2, p3.path
+  end
+
+  PATH_CHECK_H_REGEXP = Regexp.compile('^/path/to/file\.[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}.log$')
+  PATH_CHECK_H_REGEXP2 = Regexp.compile('^/PATH/TO/FILE\.[0-9A-F]{8}-([0-9A-F]{4}-){3}[0-9A-F]{12}.LOG$')
+
+  def test_uuid_hostname
+    conf1 = %[
+tag test.out
+path /path/to/file.${uuid:hostname}.log
+]
+    p1, p2, p3 = create_plugin_instances(conf1)
+    assert_match PATH_CHECK_H_REGEXP, p2.path
+    assert_equal '/path/to/file.87577bd5-6d8c-5dff-8988-0bc01cb8ed53.log', p2.path
+    assert_match PATH_CHECK_H_REGEXP2, p3.path
+    assert_equal '/PATH/TO/FILE.87577BD5-6D8C-5DFF-8988-0BC01CB8ED53.LOG', p3.path
+
+    conf2 = %[
+tag test.out
+path /path/to/file.__UUID_HOSTNAME__.log
+]
+    p1, p2, p3 = create_plugin_instances(conf2)
+    assert_match PATH_CHECK_H_REGEXP, p2.path
+    assert_equal '/path/to/file.87577bd5-6d8c-5dff-8988-0bc01cb8ed53.log', p2.path
+    assert_match PATH_CHECK_H_REGEXP2, p3.path
+    assert_equal '/PATH/TO/FILE.87577BD5-6D8C-5DFF-8988-0BC01CB8ED53.LOG', p3.path
+  end
+
+  PATH_CHECK_T_REGEXP = Regexp.compile('^/path/to/file\.[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}.out$')
+  PATH_CHECK_T_REGEXP2 = Regexp.compile('^/PATH/TO/FILE\.[0-9A-F]{8}-([0-9A-F]{4}-){3}[0-9A-F]{12}.OUT$')
+
+  def test_uuid_hostname
+    conf1 = %[
+tag test.out
+path /path/to/file.${uuid:timestamp}.out
+]
+    p1, p2, p3 = create_plugin_instances(conf1)
+    assert_match PATH_CHECK_T_REGEXP, p2.path
+    assert_match PATH_CHECK_T_REGEXP2, p3.path
+
+    conf2 = %[
+tag test.out
+path /path/to/file.__UUID_HOSTNAME__.out
+]
+    p1, p2, p3 = create_plugin_instances(conf2)
+    assert_match PATH_CHECK_T_REGEXP, p2.path
+    assert_match PATH_CHECK_T_REGEXP2, p3.path
+  end
+
+  def test_nested
+    conf = %[
+hostname test.host.local
+tag test.out
+path /path/to/file.log
+<config ${hostname}>
+  var val1.${uuid:hostname}
+  <group>
+    field value.1.${uuid:hostname}
+  </group>
+  <group>
+    field value.2.__UUID_HOSTNAME__
+  </group>
+</config>
+]
+    require 'uuidtools'
+    uuid = UUIDTools::UUID.sha1_create(UUIDTools::UUID_DNS_NAMESPACE, "test.host.local").to_s
+
+    p1, p2, p3 = create_plugin_instances(conf)
+    assert_equal "config", p3.conf.elements.first.name
+    assert_equal "test.host.local", p3.conf.elements.first.arg
+    assert_equal "val1." + uuid, p3.conf.elements.first['var']
+    assert_equal "group", p3.conf.elements.first.elements[0].name
+    assert_equal "value.1." + uuid, p3.conf.elements.first.elements[0]['field']
+    assert_equal "value.2." + uuid, p3.conf.elements.first.elements[1]['field']
+  end
+end

data/test/plugin.rb

@@ -0,0 +1,53 @@
+class Fluent::ConfigPlaceholdersTestXInput < Fluent::Input
+  Fluent::Plugin.register_input('config_placeholder_test_x', self)
+
+  attr_accessor :conf
+  def configure(conf)
+    super
+
+    @conf = conf
+  end
+end
+
+class Fluent::ConfigPlaceholdersTest0Input < Fluent::Input
+  Fluent::Plugin.register_input('config_placeholder_test_0', self)
+
+  config_param :tag, :string
+  config_param :path, :string
+
+  attr_accessor :conf
+
+  def configure(conf)
+    super
+
+    @conf = conf
+  end
+end
+
+class Fluent::ConfigPlaceholdersTest1Input < Fluent::Input
+  Fluent::Plugin.register_input('config_placeholder_test_1', self)
+
+  config_param :tag, :string
+  config_param :path, :string
+
+  include Fluent::Mixin::ConfigPlaceholders
+end
+
+class Fluent::ConfigPlaceholdersTest2Input < Fluent::Input
+  Fluent::Plugin.register_input('config_placeholder_test_2', self)
+
+  config_param :tag, :string
+  config_param :path, :string
+
+  include Fluent::Mixin::ConfigPlaceholders
+
+  attr_accessor :conf
+
+  def configure(conf)
+    super
+
+    @path.upcase!
+
+    @conf = conf
+  end
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: fluent-mixin-config-placeholders
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- TAGOMORI Satoshi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: to add various placeholders for plugin configurations
+email:
+- tagomoris@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-mixin-config-placeholders.gemspec
+- lib/fluent/mixin/config_placeholders.rb
+- test/helper.rb
+- test/mixin/test_config_placeholders.rb
+- test/plugin.rb
+homepage: https://github.com/tagomoris/fluent-mixin-config-placeholders
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: Configuration syntax extension mixin for fluentd plugin
+test_files:
+- test/helper.rb
+- test/mixin/test_config_placeholders.rb
+- test/plugin.rb