fluent-plugin-rewrite 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundler

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-rewrite.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,2 @@
+Copyright (c) 2012 Kentaro Kuribayashi
+Apache License, Version 2.0

data/README.md

@@ -0,0 +1,178 @@
+# fluent-plugin-rewrite
+
+## Component
+
+### RewriteOutput
+
+Output plugin to rewrite messages' tags/values along with pattern
+matching and re-emit them.
+
+## Synopsis
+
+```
+<match apache.log.**>
+  type rewrite
+
+  remove_prefix apache.log
+  add_prefix    filtered
+
+  <rule>
+    key     path
+    pattern \\?.+$
+    replace
+  </rule>
+  <rule>
+    key     path
+    pattern (/[^/]+)\\?([^=]+)=(\\d)
+    replace \\1/\\2/\\3
+  </rule>
+  <rule>
+    key     status
+    pattern ^500$
+    ignore  true
+  </rule>
+  <rule>
+    key           path
+    pattern       ^\/(users|entries)
+    append_to_tag true
+    fallback      others
+  </rule>
+</match>
+```
+
+## Configuration
+
+### remove_prefix / add_prefix
+
+```
+remove_prefix apache.log
+add_prefix    filtered
+```
+
+- remove_prefix: removes the string from a prefix of tag.
+- add_prefix: prepend the string to a tag.
+
+### rule: replace
+
+For example, if you want to filter out query string form URL string:
+
+```
+<rule>
+  key     path
+  pattern \\?.+$
+  replace
+</rule>
+```
+
+It executes pattern matching against a value related with `key` and replaces it with empty string if it matches.
+
+```
+/foo?bar=baz -> /foo
+```
+
+This time, if you want to rewrite path string along with some pattern:
+
+```
+<rule>
+  key     path
+  pattern (/[^/]+)\\?([^=]+)=(\\d)
+  replace \\1/\\2/\\3
+</rule>
+```
+
+It executes pattern matching against a value related with `key` and replaces it with `replace` if it matches.
+
+```
+/foo?bar=1 -> /foo/bar/1
+```
+
+### rule: ignore
+
+For example, if you want to skip a message which matches some pattern:
+
+```
+<rule>
+  key     status
+  pattern ^500$
+  ignore  true
+</rule>
+```
+
+It executes pattern matching against a value related with `key` and skip emitting the message if it matches.
+
+### rule: append_to_tag
+
+```
+<rule>
+  key           path
+  pattern       ^\/(users|entries)
+  append_to_tag true
+</rule>
+```
+
+It executes pattern matching against a value related with `key` and append mathed strings to message tag. For example:
+
+```
+apache.log { "path" : "/users/antipop" }
+```
+
+the messabe above will be re-emmited as the message below:
+
+```
+apache.log.users { "path" : "/users/antipop" }
+```
+
+If you set `fallback` option like below:
+
+```
+<rule>
+  key           path
+  pattern       ^\/(users|entries)
+  append_to_tag true
+  fallback      others
+</rule>
+```
+
+the value of `fallback` option will be appended to message tag.
+
+```
+apache.log { "path" : "/foo/bar" }
+```
+
+This time, the messabe above will be re-emmited as the message below:
+
+```
+apache.log.others { "path" : "/foo/bar" }
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-rewrite'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-rewrite
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright
+
+### Copyright
+
+Copyright (c) 2012- Kentaro Kuribayashi (@kentaro)
+
+### License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-rewrite.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-rewrite"
+  gem.version       = '0.0.1'
+  gem.authors       = ["Kentaro Kuribayashi"]
+  gem.email         = ["kentarok@gmail.com"]
+  gem.homepage      = "http://github.com/kentaro/fluent-plugin-rewrite"
+  gem.description   = %q{Fluentd plugin to rewrite tags/values along with pattern matching and re-emit them.}
+  gem.summary       = %q{Fluentd plugin to rewrite tags/values along with pattern matching and re-emit them.}
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "fluentd"
+  gem.add_runtime_dependency     "fluentd"
+end

data/lib/fluent/plugin/rewrite.rb

@@ -0,0 +1,99 @@
+module Fluent
+  class RewriteOutput < Output
+    Fluent::Plugin.register_output('rewrite', self)
+
+    config_param :remove_prefix, :string, :default => nil
+    config_param :add_prefix,    :string, :default => nil
+
+    attr_reader  :rules
+
+    def configure(conf)
+      super
+
+      if @remove_prefix
+        @removed_prefix_string = @remove_prefix + '.'
+        @removed_length = @removed_prefix_string.length
+      end
+      if @add_prefix
+        @added_prefix_string = @add_prefix + '.'
+      end
+
+      @rules = conf.elements.select { |element|
+        element.name == 'rule'
+      }.each { |element|
+        if element.has_key?("pattern")
+          element["regex"] = Regexp.new(element["pattern"])
+        end
+      }
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each do |time, record|
+        tag, record = rewrite(tag, record)
+        Engine.emit(tag, time, record) if tag && record
+      end
+
+      chain.next
+    end
+
+    def length_will_be_removed
+      return 0 unless @remove_prefix
+      (@remove_prefix + '.').length
+    end
+
+    def rewrite(tag, record)
+      if @remove_prefix and
+          ((tag.start_with?(@removed_prefix_string) && tag.length > @removed_length) ||
+          tag == @remove_prefix)
+        tag = tag[@removed_length..-1]
+      end
+
+      if @add_prefix
+        tag = tag && tag.length > 0 ? @added_prefix_string + tag : @add_prefix
+      end
+
+      rules.each do |rule|
+        tag, record = apply_rule(rule, tag, record)
+        return if !tag && !record
+      end
+
+      [tag, record]
+    end
+
+    def apply_rule(rule, tag, record)
+      tag     = rule["append_to_tag"] ? tag.dup : tag
+      key     = rule["key"]
+      pattern = rule["pattern"]
+
+      return [tag, record] if !key || !record.has_key?(key)
+      return [tag, record] unless pattern
+
+      if matched = record[key].match(rule["regex"])
+        return if rule["ignore"]
+
+        if rule["replace"]
+          replace = rule["replace"].to_s
+          record[key] = record[key].gsub(rule["regex"], replace)
+        end
+
+        if rule["append_to_tag"]
+          matched.captures.each { |m| tag << ".#{m}" }
+        end
+      else
+        if rule["append_to_tag"] && rule["fallback"]
+          tag << ".#{rule["fallback"]}"
+        end
+      end
+
+      [tag, record]
+    end
+  end
+end

data/test/plugin/test_out_rewrite.rb

@@ -0,0 +1,233 @@
+require 'test_helper'
+
+class RewriteOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(Fluent::RewriteOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      remove_prefix test
+      add_prefix    filtered
+
+      <rule>
+        key foo
+      </rule>
+      <to_be_ignored>
+        key bar
+      </to_be_ignored>
+      <rule>
+        key baz
+      </rule>
+    ])
+
+    assert_equal "test",     d.instance.remove_prefix
+    assert_equal "filtered", d.instance.add_prefix
+    assert_equal 2, d.instance.rules.size
+  end
+
+  def test_rewrite_replace
+    d1 = create_driver(%[
+      <rule>
+        key     path
+        pattern \\?.+$
+        replace
+      </rule>
+    ])
+
+    assert_equal(
+      [ "test", { "path" => "/foo" } ],
+      d1.instance.rewrite("test", { "path" => "/foo?bar=1" })
+    )
+
+    d2 = create_driver(%[
+      <rule>
+        key     path
+        pattern (/[^/]+)\\?([^=]+)=(\\d)
+        replace \\1/\\2/\\3
+      </rule>
+    ])
+
+    assert_equal(
+      [ "test", { "path" => "/foo/bar/1" } ],
+      d2.instance.rewrite("test", { "path" => "/foo?bar=1" })
+    )
+  end
+
+  def test_rewrite_ignore
+    d1 = create_driver(%[
+      <rule>
+        key     status
+        pattern ^500$
+        ignore  true
+      </rule>
+    ])
+
+    assert_equal(
+      nil,
+      d1.instance.rewrite("test", { "status" => "500" })
+    )
+
+    d2 = create_driver(%[
+      <rule>
+        key     status
+        pattern ^(?!200)\\d+$
+        ignore  true
+      </rule>
+    ])
+
+    assert_equal(
+      [ "test", { "status" => "200" } ],
+      d2.instance.rewrite("test", { "status" => "200" })
+    )
+    %w[301 404 500].each do |status|
+      assert_equal(
+        nil,
+        d2.instance.rewrite("test", { "status" => status })
+      )
+    end
+
+    d3 = create_driver(%[
+      <rule>
+        key     flag
+        pattern ^$
+        ignore  true
+      </rule>
+    ])
+
+    assert_equal(
+      nil,
+      d3.instance.rewrite("test", { "flag" => "" })
+    )
+  end
+
+  def test_rewrite_append_tag
+    d1 = create_driver(%[
+      <rule>
+        key           path
+        pattern       ^\/(users|entries)
+        append_to_tag true
+      </rule>
+    ])
+
+    assert_equal(
+      [ "test.users", { "path" => "/users/antipop" } ],
+      d1.instance.rewrite("test", { "path" => "/users/antipop" })
+    )
+    assert_equal(
+      [ "test", { "path" => "/unmatched/path" } ],
+      d1.instance.rewrite("test", { "path" => "/unmatched/path" })
+    )
+
+    d2 = create_driver(%[
+      <rule>
+        key           path
+        pattern       ^\/(users|entries)
+        append_to_tag true
+        fallback      others
+      </rule>
+    ])
+
+    assert_equal(
+      [ "test.users", { "path" => "/users/antipop" } ],
+      d2.instance.rewrite("test", { "path" => "/users/antipop" })
+    )
+    assert_equal(
+      [ "test.others", { "path" => "/unmatched/path" } ],
+      d2.instance.rewrite("test", { "path" => "/unmatched/path" })
+    )
+  end
+
+  def test_rewrite_rules
+    d = create_driver(%[
+      <rule>
+        key     path
+        pattern \\?.+$
+        replace
+      </rule>
+      <rule>
+        key     status
+        pattern ^500$
+        ignore  true
+      </rule>
+      <rule>
+        key           path
+        pattern       ^\/(users|entries)
+        append_to_tag true
+        fallback      others
+      </rule>
+    ])
+
+    assert_equal(
+      [ "test.others", { "path" => "/foo" } ],
+      d.instance.rewrite("test", { "path" => "/foo?bar=1" })
+    )
+    assert_equal(
+      [ "test.users", { "path" => "/users/antipop" } ],
+      d.instance.rewrite("test", { "path" => "/users/antipop?hoge=1" })
+    )
+    assert_equal(
+      nil,
+      d.instance.rewrite("test", { "path" => "/foo?bar=1", "status" => "500" })
+    )
+  end
+
+  def test_tag_prefix
+    d = create_driver(%[
+      remove_prefix test
+      add_prefix    filtered
+
+      <rule>
+        key           path
+        pattern       ^\/(users|entries)
+        append_to_tag true
+      </rule>
+    ])
+
+    assert_equal(
+      [ "filtered.users", { "path" => "/users/antipop" } ],
+      d.instance.rewrite("test", { "path" => "/users/antipop" })
+    )
+  end
+
+  def test_emit
+    d = create_driver(%[
+      remove_prefix test
+      add_prefix    filtered
+
+      <rule>
+        key     path
+        pattern \\?.+$
+        replace
+      </rule>
+      <rule>
+        key     status
+        pattern ^500$
+        ignore  true
+      </rule>
+      <rule>
+        key           path
+        pattern       ^\/(users|entries)
+        append_to_tag true
+        fallback      others
+      </rule>
+    ])
+
+    d.run do
+      d.emit({ "path" => "/foo?bar=1" })
+      d.emit({ "path" => "/users/antipop?hoge=1" })
+      d.emit({ "path" => "/foo?bar=1", "status" => "500" })
+    end
+    emits = d.emits
+
+    assert_equal 2, emits.size
+    assert_equal('filtered.others', emits[0][0])
+    assert_equal({ "path" => "/foo" }, emits[0][2])
+    assert_equal('filtered.users', emits[1][0])
+    assert_equal({ "path" => "/users/antipop" }, emits[1][2])
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/rewrite'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-rewrite
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Kentaro Kuribayashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to rewrite tags/values along with pattern matching and
+  re-emit them.
+email:
+- kentarok@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-rewrite.gemspec
+- lib/fluent/plugin/rewrite.rb
+- test/plugin/test_out_rewrite.rb
+- test/test_helper.rb
+homepage: http://github.com/kentaro/fluent-plugin-rewrite
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluentd plugin to rewrite tags/values along with pattern matching and re-emit
+  them.
+test_files:
+- test/plugin/test_out_rewrite.rb
+- test/test_helper.rb
+has_rdoc: