fluent-plugin-extract_query_params 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundle

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,2 @@
+Copyright (c) 2012 Kentaro Kuribayashi
+Apache License, Version 2.0

data/README.md

@@ -0,0 +1,93 @@
+# fluent-plugin-extract_query_params
+
+## Component
+
+### ExtractQueryParamsOutput
+
+Fluentd plugin to extract key/values from URL query parameters.
+
+## Synopsis
+
+Imagin you have a config as below:
+
+```
+<match test.**>
+  type extract_query_params
+
+  key            url
+  add_tag_prefix extracted.
+  only           foo, baz
+</match>
+```
+
+And you feed such a value into fluentd:
+
+```
+"test" => {
+  "url" => "http://example.com/?foo=bar&baz=qux&hoge=fuga"
+}
+```
+
+Then you'll get re-emmited tag/record-s below:
+
+```
+"extracted.test" => {
+  "url" => "http://example.com/?foo=bar&baz=qux&hoge=fuga"
+  "foo" => "bar",
+  "baz" => "quex"
+}
+```
+
+`hoge` parameter is not extracted.
+
+## Configuration
+
+### key
+
+`key` is used to point a key whose value contains URL string.
+
+### remove_tag_prefix, remove_tag_suffix, add_tag_prefix, add_tag_suffix
+
+These params are included from `Fluent::HandleTagNameMixin`. See the code for details.
+
+You must add at least one of these params.
+
+### only
+
+If set, only the key/value whose key is included `only` will be added to the record.
+
+### except
+
+If set, the key/value whose key is included `except` will NOT be added to the record.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-extract_query_params'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-extract_query_params
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright
+
+### Copyright
+
+Copyright (c) 2013- Kentaro Kuribayashi (@kentaro)
+
+### License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-extract_query_params.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-extract_query_params'
+  gem.version       = '0.0.1'
+  gem.authors       = ['Kentaro Kuribayashi']
+  gem.email         = ['kentarok@gmail.com']
+  gem.homepage      = 'http://github.com/kentaro/fluent-plugin-extract_query_params'
+  gem.description   = %q{Fluentd plugin to extract key/values from URL query parameters.}
+  gem.summary       = %q{Fluentd plugin to extract key/values from URL query parameters}
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'fluentd'
+  gem.add_runtime_dependency     'fluentd'
+end

data/lib/fluent/plugin/out_extract_query_params.rb

@@ -0,0 +1,67 @@
+require 'uri'
+
+module Fluent
+  class ExtractQueryParamsOutput < Output
+    include Fluent::HandleTagNameMixin
+
+    Fluent::Plugin.register_output('extract_query_params', self)
+
+    config_param :key,    :string
+    config_param :only,   :string, :default => nil
+    config_param :except, :string, :default => nil
+
+    def configure(conf)
+      super
+
+      if (
+          !remove_tag_prefix &&
+          !remove_tag_suffix &&
+          !add_tag_prefix    &&
+          !add_tag_suffix
+      )
+        raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
+      end
+
+      @include_keys = only   && only.split(/\s*,\s*/).inject({}) do |hash, i|
+        hash[i] = true
+        hash
+      end
+      @exclude_keys = except && except.split(/\s*,\s*/).inject({}) do |hash, i|
+        hash[i] = true
+        hash
+      end
+    end
+
+    def emit(tag, es, chain)
+      es.each do |time, record|
+        filter_record(tag, time, record)
+        Engine.emit(tag, time, record)
+      end
+
+      chain.next
+    end
+
+    def filter_record(tag, time, record)
+      if record[key]
+        begin
+          url = URI.parse(record[key])
+          url.query.split('&').each do |pair|
+            key, value = pair.split('=').map { |i| URI.unescape(i) }
+
+            if only
+              record[key] = value if @include_keys.has_key?(key)
+            elsif except
+              record[key] = value if !@exclude_keys.has_key?(key)
+            else
+              record[key] = value
+            end
+          end
+        rescue URI::InvalidURIError => error
+          $log.warn("out_extract_query_params: #{error.message}")
+        end
+      end
+
+      super(tag, time, record)
+    end
+  end
+end

data/test/plugin/test_out_extract_query_params.rb

@@ -0,0 +1,131 @@
+# -*- encoding: utf-8 -*-
+
+require 'test_helper'
+
+class ExtractQueryParamsOutputTest < Test::Unit::TestCase
+  URL = 'http://example.com/?foo=bar&baz=qux&%E3%83%A2%E3%83%AA%E3%82%B9=%E3%81%99%E3%81%9F%E3%81%98%E3%81%8A'
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(
+      Fluent::ExtractQueryParamsOutput, tag
+    ).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      key            url
+      add_tag_prefix extracted.
+      only           foo, baz
+    ])
+
+    assert_equal 'url',        d.instance.key
+    assert_equal 'extracted.', d.instance.add_tag_prefix
+    assert_equal 'foo, baz',   d.instance.only
+
+    # when mandatory keys not set
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        key foo
+      ])
+    end
+  end
+
+  def test_filter_record
+    d = create_driver(%[
+      key            url
+      add_tag_prefix extracted.
+    ])
+
+    tag    = 'test'
+    record = {
+      'url' => URL,
+    }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal URL,       record['url']
+    assert_equal 'bar',     record['foo']
+    assert_equal 'qux',     record['baz']
+    assert_equal 'すたじお', record['モリス']
+  end
+
+  def test_filter_record_with_only
+    d = create_driver(%[
+      key            url
+      add_tag_prefix extracted.
+      only           foo, baz
+    ])
+
+    tag    = 'test'
+    record = { 'url' => URL }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal URL,   record['url']
+    assert_equal 'bar', record['foo']
+    assert_equal 'qux', record['baz']
+    assert_nil record['モリス']
+  end
+
+  def test_filter_record_with_except
+    d = create_driver(%[
+      key            url
+      add_tag_prefix extracted.
+      except         baz, モリス
+    ])
+
+    tag    = 'test'
+    record = { 'url' => URL }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal URL,   record['url']
+    assert_equal 'bar', record['foo']
+    assert_nil record['baz']
+    assert_nil record['モリス']
+  end
+
+  def test_emit
+    d = create_driver(%[
+      key            url
+      add_tag_prefix extracted.
+      only           foo, baz
+    ])
+    d.run { d.emit('url' => URL) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'extracted.test', emits[0][0]
+    assert_equal URL,              emits[0][2]['url']
+    assert_equal 'bar',            emits[0][2]['foo']
+    assert_equal 'qux',            emits[0][2]['baz']
+  end
+
+  def test_emit_without_match_key
+    d = create_driver(%[
+      key            no_such_key
+      add_tag_prefix extracted.
+      only           foo, baz
+    ])
+    d.run { d.emit('url' => URL) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'extracted.test', emits[0][0]
+    assert_equal URL,              emits[0][2]['url']
+  end
+
+  def test_emit_with_invalid_url
+    d = create_driver(%[
+      key            url
+      add_tag_prefix extracted.
+    ])
+    d.run { d.emit('url' => 'invalid url') }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'extracted.test', emits[0][0]
+    assert_equal 'invalid url',    emits[0][2]['url']
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_extract_query_params'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-extract_query_params
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Kentaro Kuribayashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to extract key/values from URL query parameters.
+email:
+- kentarok@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-extract_query_params.gemspec
+- lib/fluent/plugin/out_extract_query_params.rb
+- test/plugin/test_out_extract_query_params.rb
+- test/test_helper.rb
+homepage: http://github.com/kentaro/fluent-plugin-extract_query_params
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluentd plugin to extract key/values from URL query parameters
+test_files:
+- test/plugin/test_out_extract_query_params.rb
+- test/test_helper.rb