fluent-plugin-parse_request_body 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e3d528441f6fd97325700bfddb162787bd74740b4c2daf677d1f8d136b7234da
+  data.tar.gz: e860534ad5a27a7784efd9482910eec6debd6d5c43af887af9c1b7f3de0c6def
+SHA512:
+  metadata.gz: 2b7c36a29ac960ec14ba04eede5c73e13e908ea2eeed58417569178263aefbd3f53c6e4765689408acba2ec40ad4163b5fc312cbd95bc61c14105a6333d8a318
+  data.tar.gz: 7629939eb2b5439c1a4cc918a198c7d09f1ce650a448da7a10aa3659b41d6ed9ded782e66f3b384ba2cb6eed1bfbfeae6fc695c0fab2d795d2214aad4be3ec16

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+*.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundle

data/.travis.yml

@@ -0,0 +1,13 @@
+sudo: false
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.1
+
+gemfile:
+  - gemfiles/fluentd_0.14.gemfile
+
+before_script: gem update bundler
+script: bundle exec rake test

data/Appraisals

@@ -0,0 +1,3 @@
+appraise 'fluentd-0.14' do
+  gem 'fluentd', '~> 0.14.0'
+end

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,2 @@
+Copyright (c) 2018 EkiSong
+Apache License, Version 2.0

data/README.md

@@ -0,0 +1 @@
+# fluent-plugin-parse_request_body

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-parse_request_body.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-parse_request_body'
+  gem.version       = '0.0.1'
+  gem.authors       = ['EkiSong']
+  gem.email         = ['yifriday0614@gmail.com']
+  gem.homepage      = 'https://github.com/yifriday/fluent-plugin-parse_request_body.git'
+  gem.description   = %q{Fluentd plugin to parse request body.}
+  gem.summary       = %q{Fluentd plugin to parse request body}
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  if defined?(RUBY_VERSION) && RUBY_VERSION > '2.2'
+    gem.add_development_dependency "test-unit", '~> 3'
+  end
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'appraisal'
+  gem.add_runtime_dependency     'fluentd', ['>= 0.14.8', '< 2']
+end

data/gemfiles/fluentd_0.14.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "fluentd", "~> 0.14.0"
+
+gemspec :path => "../"

data/lib/fluent/plugin/filter_parse_request_body.rb

@@ -0,0 +1,31 @@
+require "fluent/plugin/filter"
+require "fluent/plugin/parse_request_body_extractor"
+
+module Fluent::Plugin
+  class ParseRequestBodyFilter < Fluent::Plugin::Filter
+
+    Fluent::Plugin.register_filter('parse_request_body', self)
+
+    desc "point a key whose value contains URL string."
+    config_param :key,    :string
+    desc "If set, only the key/value whose key is included only will be added to the record."
+    config_param :only,   :string, default: nil
+    desc "If set, the key/value whose key is included except will NOT be added to the record."
+    config_param :except, :string, default: nil
+    desc "If set to true, the original key url will be discarded from the record."
+    config_param :discard_key, :bool, default: false
+    desc "Prefix of fields."
+    config_param :add_field_prefix, :string, default: nil
+    desc "If set to true, permit blank key."
+    config_param :permit_blank_key, :bool, default: false
+
+    def configure(conf)
+      super
+      @extractor = Fluent::Plugin::ParseRequestBodyExtractor.new(self, conf)
+    end
+
+    def filter(tag, time, record)
+      @extractor.add_query_params_field(record)
+    end
+  end
+end

data/lib/fluent/plugin/out_parse_request_body.rb

@@ -0,0 +1,47 @@
+require "fluent/plugin/output"
+require "fluent/plugin/parse_request_body_extractor"
+
+module Fluent::Plugin
+  class ParseRequestBodyOutput < Fluent::Plugin::Output
+    include Fluent::HandleTagNameMixin
+
+    Fluent::Plugin.register_output('parse_request_body', self)
+
+    helpers :event_emitter
+
+    desc "point a key whose value contains URL string."
+    config_param :key,    :string
+    desc "If set, only the key/value whose key is included only will be added to the record."
+    config_param :only,   :string, default: nil
+    desc "If set, the key/value whose key is included except will NOT be added to the record."
+    config_param :except, :string, default: nil
+    desc "If set to true, the original key url will be discarded from the record."
+    config_param :discard_key, :bool, default: false
+    desc "Prefix of fields."
+    config_param :add_field_prefix, :string, default: nil
+    desc "If set to true, permit blank key."
+    config_param :permit_blank_key, :bool, default: false
+
+    def configure(conf)
+      super
+      @extractor = Fluent::Plugin::ParseRequestBodyExtractor.new(self, conf)
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def filter_record(tag, time, record)
+      record = @extractor.add_query_params_field(record)
+      super(tag, time, record)
+    end
+
+    def process(tag, es)
+      es.each do |time, record|
+        t = tag.dup
+        filter_record(t, time, record)
+        router.emit(t, time, record)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parse_request_body_extractor.rb

@@ -0,0 +1,92 @@
+require 'uri'
+require 'cgi/util'
+require 'webrick'
+
+module Fluent::Plugin
+  class ParseRequestBodyExtractor
+
+    attr_reader :log
+
+    def initialize(plugin, conf)
+      @log = plugin.log
+
+      if plugin.is_a?(Fluent::Plugin::Output)
+        unless have_tag_option?(plugin)
+          raise Fluent::ConfigError, "out_parse_request_body: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
+        end
+      end
+
+      @key = plugin.key
+      @only = plugin.only
+      @except = plugin.except
+      @discard_key = plugin.discard_key
+      @add_field_prefix = plugin.add_field_prefix
+      @permit_blank_key = plugin.permit_blank_key
+
+      if @only
+        @include_keys = @only.split(/\s*,\s*/).inject({}) do |hash, i|
+          hash[i] = true
+          hash
+        end
+      end
+
+      if @except
+        @exclude_keys = @except.split(/\s*,\s*/).inject({}) do |hash, i|
+          hash[i] = true
+          hash
+        end
+      end
+    end
+
+    def add_query_params_field(record)
+      return record unless record[@key]
+      add_query_params(record[@key], record)
+      record.delete(@key) if @discard_key
+      record
+    end
+
+    private
+
+    def have_tag_option?(plugin)
+      plugin.remove_tag_prefix ||
+        plugin.remove_tag_suffix ||
+        plugin.add_tag_prefix    ||
+        plugin.add_tag_suffix
+    end
+
+    def create_field_key(field_key)
+      if add_field_prefix?
+        "#{@add_field_prefix}#{field_key}"
+      else
+        field_key
+      end
+    end
+
+    def add_field_prefix?
+      !!@add_field_prefix
+    end
+
+    def permit_blank_key?
+      @permit_blank_key
+    end
+
+    def add_query_params(body, record)
+      return if body.nil?
+      body.split('&').each do |pair|
+        key, value = pair.split('=', 2).map { |i| CGI.unescape(i) }
+        next if (key.nil? || key.empty?) && (!permit_blank_key? || value.nil? || value.empty?)
+        key ||= ''
+        value ||= ''
+
+        key = create_field_key(key)
+        if @only
+          record[key] = value if @include_keys.has_key?(key)
+        elsif @except
+          record[key] = value if !@exclude_keys.has_key?(key)
+        else
+          record[key] = value
+        end
+      end
+    end
+  end
+end

data/test/plugin/test_filter_parse_request_body.rb

@@ -0,0 +1,162 @@
+require 'test_helper'
+require 'fluent/plugin/filter_parse_request_body'
+
+class ParseRequestBodyFilterTest < Test::Unit::TestCase
+  URL = 'http://example.com:80'
+  BODY_ONLY = {
+      'uid' => 123456,
+      'sid' => '123456fjkjafkjadk',
+      'location' => '134.234,144.333'
+    }
+
+  def setup
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Filter.new(
+      Fluent::Plugin::ParseRequestBodyFilter
+    ).configure(conf)
+  end
+
+  def filter(config, messages)
+    d = create_driver(config)
+    d.run(default_tag: "test") {
+      messages.each {|message|
+        d.feed(@time, message)
+      }
+    }
+    d.filtered_records
+  end
+
+  def test_configure
+    d = create_driver(%[
+      key            request_body
+      only           uid, sid
+    ])
+
+    assert_equal 'request_body',        d.instance.key
+    assert_equal 'uid, sid',   d.instance.only
+  end
+
+  def test_filter
+    config = %[
+      key            request_body
+    ]
+
+    record = {
+      'request_body' => BODY_ONLY,
+    }
+    expected = {
+      'request_body' => BODY_ONLY,
+      'location' => '134.234,144.333'
+    }
+    filtered = filter(config, [record])
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_with_field_prefix
+    config = %[
+      key            request_body
+      add_field_prefix query_
+    ]
+
+    record = {
+      'request_body' => BODY_ONLY,
+    }
+    expected = {
+      'request_body' => BODY_ONLY,
+      'query_uid' => 123456,
+      'query_sid' => '123456fjkjafkjadk',
+      'query_location' => '134.234,144.333'
+    }
+    filtered = filter(config, [record])
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_with_only
+    config = %[
+      key            request_body
+      only           uid, sid
+    ]
+
+    record = { 'request_body' => BODY_ONLY }
+    expected = {
+      'request_body' => BODY_ONLY,
+      'uid' => 123456,
+      'sid' => '123456fjkjafkjadk',
+    }
+    filtered = filter(config, [record])
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_with_except
+    config = %[
+      key            request_body
+      except         uid
+    ]
+
+    record = { 'request_body' => BODY_ONLY }
+    expected = {
+      'sid' => '123456fjkjafkjadk',
+      'location' => '134.234,144.333',
+    }
+    filtered = filter(config, [record])
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_with_discard
+    config = %[
+      key            url
+      discard_key true
+    ]
+
+    record = { 'request_body' => BODY_ONLY }
+    expected = {
+      'uid' => 123456,
+      'sid' => '123456fjkjafkjadk',
+      'location' => '134.234,144.333'
+    }
+    filtered = filter(config, [record])
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_multi_records
+    config = %[
+      key            request_body
+      only           uid, sid
+    ]
+    records = [
+      { 'request_body' => BODY_ONLY },
+      { 'request_body' => BODY_ONLY },
+      { 'request_body' => BODY_ONLY }
+    ]
+    expected = [
+      { 'request_body' => BODY_ONLY, 'uid' => 123456, 'sid' => '123456fjkjafkjadk' },
+      { 'request_body' => BODY_ONLY, 'uid' => 123456, 'sid' => '123456fjkjafkjadk' },
+      { 'request_body' => BODY_ONLY, 'uid' => 123456, 'sid' => '123456fjkjafkjadk' }
+    ]
+    filtered = filter(config, records)
+    assert_equal(expected, filtered)
+  end
+
+  def test_emit_without_match_key
+    config = %[
+      key            no_such_key
+      only           uid, sid
+    ]
+    record = { 'request_body' => BODY_ONLY }
+    filtered = filter(config, [record])
+    assert_equal(record, filtered[0])
+  end
+
+  def test_emit_with_invalid_url
+    config = %[
+      key            request_body
+    ]
+    record = { 'request_body' => BODY_ONLY }
+    filtered = filter(config, [record])
+    assert_equal([record], filtered)
+  end
+end

data/test/plugin/test_out_parse_request_body.rb

@@ -0,0 +1,193 @@
+require 'test_helper'
+require 'fluent/plugin/out_parse_request_body'
+
+class ParseRequestBodyOutputTest < Test::Unit::TestCase
+  URL = 'http://example.com:80'
+  BODY_ONLY = {
+      'uid' => 123456,
+      'sid' => '123456fjkjafkjadk',
+      'location' => '134.234,144.333'
+    }
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(
+      Fluent::Plugin::ParseRequestBodyOutput
+    ).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+      only           uid, sid
+    ])
+
+    assert_equal 'request_body',        d.instance.key
+    assert_equal 'extracted.', d.instance.add_tag_prefix
+    assert_equal 'uid, sid',   d.instance.only
+
+    # when mandatory keys not set
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        key uid
+      ])
+    end
+  end
+
+  def test_filter_record
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+    ])
+
+    record = {
+      'request_body' => BODY_ONLY,
+    }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal BODY_ONLY,               record['request_body']
+    assert_equal 123456,                  record['uid']
+    assert_equal '123456fjkjafkjadk',     record['sid']
+    assert_equal '134.234,144.333',       record['location']
+  end
+
+  def test_filter_record_with_field_prefix
+    d = create_driver(%[
+      key            request_body
+      add_field_prefix query_
+      add_tag_prefix extracted.
+    ])
+
+    record = {
+      'request_body' => BODY_ONLY,
+    }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal BODY_ONLY,       record['request_body']
+    assert_nil record['uid']
+    assert_nil record['sid']
+    assert_nil record['location']
+    assert_equal 123456,     record['query_uid']
+    assert_equal '123456fjkjafkjadk',     record['query_sid']
+    assert_equal '134.234,144.333', record['query_location']
+  end
+
+  def test_filter_record_with_only
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+      only           uid, sid
+    ])
+
+    record = { 'request_body' => BODY_ONLY }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal BODY_ONLY,   record['request_body']
+    assert_equal 123456, record['uid']
+    assert_equal '123456fjkjafkjadk', record['sid']
+    assert_nil record['location']
+  end
+
+  def test_filter_record_with_except
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+      except         baz, モリス
+    ])
+
+    record = { 'request_body' => BODY_ONLY }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal BODY_ONLY,   record['request_body']
+    assert_equal 123456, record['uid']
+    assert_nil record['sid']
+    assert_nil record['location']
+  end
+
+  def test_filter_record_with_discard
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+      discard_key true
+    ])
+
+    record = { 'request_body' => BODY_ONLY }
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_nil               record['nil']
+    assert_nil               record['request_body']
+    assert_equal 123456,      record['uid']
+    assert_equal '123456fjkjafkjadk',      record['sid']
+    assert_equal '134.234,144.333', record['location']
+  end
+
+  def test_emit
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+      only           uid, sid
+    ])
+    d.run(default_tag: "test") { d.feed('request_body' => BODY_ONLY) }
+    events = d.events
+
+    assert_equal 1, events.count
+    assert_equal 'extracted.test', events[0][0]
+    assert_equal BODY_ONLY,              events[0][2]['request_body']
+    assert_equal 123456,            events[0][2]['uid']
+    assert_equal '123456fjkjafkjadk',            events[0][2]['sid']
+  end
+
+  def test_emit_multi
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+      only           uid, sid
+    ])
+    d.run(default_tag: "test") do
+      d.feed('request_body' => BODY_ONLY)
+      d.feed('request_body' => BODY_ONLY)
+      d.feed('request_body' => BODY_ONLY)
+    end
+    events = d.events
+
+    assert_equal 3, events.count
+
+    events.each do |e|
+      assert_equal 'extracted.test', e[0]
+      assert_equal BODY_ONLY,              e[2]['request_body']
+      assert_equal 123456,            e[2]['uid']
+      assert_equal '123456fjkjafkjadk',            e[2]['sid']
+    end
+  end
+
+  def test_emit_without_match_key
+    d = create_driver(%[
+      key            no_such_key
+      add_tag_prefix extracted.
+      only           uid, sid
+    ])
+    d.run(default_tag: "test") { d.feed('request_body' => BODY_ONLY) }
+    events = d.events
+
+    assert_equal 1, events.count
+    assert_equal 'extracted.test', events[0][0]
+    assert_equal BODY_ONLY,              events[0][2]['request_body']
+  end
+
+  def test_emit_with_invalid_url
+    d = create_driver(%[
+      key            request_body
+      add_tag_prefix extracted.
+    ])
+    d.run(default_tag: "test") { d.feed('request_body' => 'invalid url') }
+    events = d.events
+
+    assert_equal 1, events.count
+    assert_equal 'extracted.test', events[0][0]
+    assert_equal 'invalid url',    events[0][2]['request_body']
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+require 'bundler/setup'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
+
+require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/driver/filter'

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-parse_request_body
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- EkiSong
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-09-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.8
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Fluentd plugin to parse request body.
+email:
+- yifriday0614@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Appraisals
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-parse_request_body.gemspec
+- gemfiles/fluentd_0.14.gemfile
+- lib/fluent/plugin/filter_parse_request_body.rb
+- lib/fluent/plugin/out_parse_request_body.rb
+- lib/fluent/plugin/parse_request_body_extractor.rb
+- test/.DS_Store
+- test/plugin/test_filter_parse_request_body.rb
+- test/plugin/test_out_parse_request_body.rb
+- test/test_helper.rb
+homepage: https://github.com/yifriday/fluent-plugin-parse_request_body.git
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to parse request body
+test_files:
+- test/.DS_Store
+- test/plugin/test_filter_parse_request_body.rb
+- test/plugin/test_out_parse_request_body.rb
+- test/test_helper.rb