fluent-plugin-split_record 0.12.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0887090cebdf0de99fa03fcd83c7702a9013cf44
+  data.tar.gz: ffd3606d5bb4a9f0a563102b1d6e39fbd0243e40
+SHA512:
+  metadata.gz: 84593232bdea75447f1d3dba891f464eca40392902918796efd28dfcadad6ee8b23d5f49a76ea909f9a3ce4e4cfd468db4dddbd1af08e70ddbe576aeacc276ac
+  data.tar.gz: 0888ea9208dad158790fce11fc0fe4168f996f754ad07af53fb1d430f8b73f96330131cf75cb7f9d982a2f2e9ca050010c7034053d90c28bcea2f0a03607d52d

data/.gitignore

@@ -0,0 +1,19 @@
+*~
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2013 Masahiro Sano
+Copyright (c) 2017 Michael Adams
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,74 @@
+# fluent-plugin-split_record
+
+Fluentd filter plugin to split a record into multiple records with key/value pair. Compatible with 0.12 and 0.14 versions of fluentd.
+
+## Overview
+This plugin splits a record and parses each results to make key/value pairs; Logstash's [kv filter](https://www.elastic.co/guide/en/logstash/current/plugins-filters-kv.html) is a good example of this. It is a successor to [fluent-plugin-split](https://github.com/kazegusuri/fluent-plugin-split/); a 0.10 output plugin. This is NOT the current 0.12+ [fluent-plugin-split](https://github.com/toyama0919/fluent-plugin-split/): that one is what currently installs with ruby-gem, and splits CSV-style content.
+
+Normally you can use a regular expression to parse a record. It is difficult to parse a record which has ambiguous numbers of data like a following record.
+
+**Before**
+```json
+{"message":"key1=val1 key2=val2 key3=val3"}
+```
+
+**After**
+```json
+{"key1":"val1","key2":"val2","key3":"val3"}
+```
+
+## Installation
+
+### Local/Build
+```
+$ git clone https://github.com/unquietwiki/fluent-plugin-split_record.git && cd fluent-plugin-split_record
+$ td-agent-gem build fluent-plugin-split_record.gemspec
+$ td-agent-gem install fluent-plugin-split_record-0.12.1.gem
+```
+
+### Online
+```
+$ td-agent-gem install fluent-plugin-split_record
+```
+
+## Configuration
+
+### Parameters
+
+|parameter|description|default|
+|---|---|---|
+|tag| key name for tag | |
+|format| regexp to parse a record after split | '(?<key>\S*)=(?<value>\S*)' |
+|substring_format| regexp used to identify substrings | '(?<key>\S*)=\\"(?<value>.*?)\\"' |
+|key_name| key name to be split | |
+|out_key| key name of json object which includes divided records | nil |
+|reserve_msg| if original message is reserved or not | nil |
+|keys_prefix| if set, all extracted keys names will be preceded by this string | nil |
+
+### Example
+
+You may want to pre-process with the [regexp parser](https://docs.fluentd.org/v0.12/articles/parser_regexp) to remove/tag other elements first; this is a requirement if working with [SonicWall syslog input](http://software.sonicwall.com/manual/232-001835-00_rev_a_sonicos_log_event_reference_guide.pdf), which is otherwise an array of key-value pairs.
+
+```
+<source>
+  @type udp
+  port 514
+  format /\<(?<prefix>[0-9]{1,3})\>(?<extradata>.+)$\z/
+  tag FW
+</source>
+
+<filter FW.**>
+  @type split_record
+  tag FW
+  key_name extradata
+  reserve_msg no
+</filter>
+```
+
+## References
+
+* https://github.com/repeatedly/fluent-plugin-record-modifier/
+* https://docs.fluentd.org/v0.12/articles/plugin-development
+* https://ruby-doc.org/core-2.1.5/
+* https://stackoverflow.com/questions/tagged/ruby
+* http://rubular.com/

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => [:test]
+

data/fluent-plugin-split_record.gemspec

@@ -0,0 +1,18 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-split_record"
+  spec.version       = "0.12.1"
+  spec.authors       = ["Masahiro Sano","Michael Adams"]
+  spec.email         = ["sabottenda@gmail.com","unquietwiki@gmail.com"]
+  spec.description   = %q{Fluentd filter plugin to split a record into multiple records with key/value pair.}
+  spec.summary       = %q{Successor to original fluent-plugin-split. Updated for fluentd 0.12 and 0.14, with non-conflicting name.}
+  spec.homepage      = "https://github.com/unquietwiki/fluent-plugin-split_record"
+  spec.license       = "MIT"
+  spec.files          = `git ls-files`.split("\n")
+  spec.test_files     = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables    = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_dependency "fluentd",">= 0.12.39","< 0.16"
+end

data/lib/fluent/plugin/filter_split_record.rb

@@ -0,0 +1,105 @@
+# Derived from https://github.com/kazegusuri/fluent-plugin-split/blob/master/lib/fluent/plugin/out_split.rb
+# Aug 30, 2017; Michael Adams; unquietwiki@gmail.com
+
+# References
+# https://github.com/repeatedly/fluent-plugin-record-modifier/
+# https://docs.fluentd.org/v0.12/articles/plugin-development
+# https://ruby-doc.org/core-2.1.5/
+# https://stackoverflow.com/questions/tagged/ruby
+
+require 'fluent/filter'
+
+module Fluent
+  class SplitRecordFilter < Filter
+    Fluent::Plugin.register_filter("split_record", self)
+
+    # Parameters
+    config_param :tag, :string
+    config_param :key_name, :string
+    config_param :out_key, :string, :default => nil
+    config_param :reserve_msg, :bool, :default => nil
+    config_param :keys_prefix, :string, :default => nil
+    config_param :format, :string, :default => '(?<key>\S*)=(?<value>\S*)'
+    config_param :substring_format, :string, :default => '(?<key>\S*)=\\"(?<value>.*?)\\"'
+    
+    # Configuration
+    def configure(conf)
+      super
+      @format_regex = Regexp.new(@format)
+      @format_regex_substring = Regexp.new(@substring_format)
+      unless @format_regex.names.include?("key") and @format_regex.names.include?("value")
+          raise ConfigError, "split_record: format must have named_captures of key and value"
+      end
+      if (!keys_prefix.nil? && keys_prefix.is_a?(String))
+        @store_fun = method(:store_with_prefix)
+      else
+        @store_fun = method(:store)
+      end
+    end
+
+    # ===== Required API methods =====
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def filter(tag, time, record)
+      record
+    end
+
+    def filter_stream(tag, es)
+      mes = MultiEventStream.new
+      es.each { |time, record|
+        begin
+          msg = record[@key_name]
+          record.delete(@key_name) unless @reserve_msg
+          data = split_message(msg)
+          if @out_key.nil?
+            record.merge!(data)
+          else
+            record[@out_key] = data
+          end
+          mes.add(time, record)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      }
+      mes
+    end
+
+    # ===== Private methods =====
+    private
+
+    # Message splitter
+    def split_message(message)
+      return {} unless message.is_a?(String)
+      # Convert key-pairs as found
+      if @format_regex_substring.nil?
+        key_values = message.scan @format_regex
+      # Pop off substrings; get their key-pairs; then scan the leftovers
+      else
+        key_values = message.scan @format_regex_substring
+        leftovers = message.gsub(@format_regex_substring,'').scan(@format_regex)
+        leftovers.each { |e| key_values << e }
+      end
+      # Store key pairs
+      data = {}
+      key_values.each { |e| @store_fun.call(data,e[0],e[1]) }
+      data
+    end
+
+    # Store key/value pair
+    def store(data, key, value)
+      data.store(key, value)
+    end
+
+    # Store key/value pair, with prefix
+    def store_with_prefix(data, key, value)
+      data.store(@keys_prefix+key, value)
+    end
+
+  end
+end

data/test/helper.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/filter_split_record'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_filter_split_record.rb

@@ -0,0 +1,109 @@
+require 'fluent/test'
+require 'fluent/plugin/filter_split_record'
+
+class SplitRecordFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type split_record
+    tag foo.filtered
+    key_name message
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test:::OutputTestDriver.new(Fluent::SplitRecordFilter, tag).configure(conf)
+  end
+
+  def get_hostname
+    require 'socket'
+    Socket.gethostname.chomp
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        format /aa/
+      ])
+    }
+    assert_raise(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        format (?<key>a)
+      ])
+    }
+    assert_raise(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        format (?<value>a)
+      ])
+    }
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        format (?<value>a) (?<key>b)
+      ])
+    }
+  end
+
+  def test_format_1
+    d = create_driver
+
+    d.run do
+      d.emit({"message" => "key1=val1 key2=val2"})
+      d.emit({"message" => " key1=val1 "})
+      d.emit({"message" => " "})
+      d.emit({"message" => 1})
+    end
+
+    mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => 'test'}
+    assert_equal [
+      {"key1" => "val1", "key2" => "val2"},
+      {"key1" => "val1"},
+      {},
+      {},
+    ], d.records
+  end
+
+  def test_format_2
+    d = create_driver(CONFIG + %[
+      separator ,
+      reserve_msg true
+    ])
+
+    d.run do
+      d.emit({"message" => "key1=val1,key2=val2"})
+    end
+
+    assert_equal [
+      {"message" => "key1=val1,key2=val2", "key1" => "val1", "key2" => "val2"},
+    ], d.records
+  end
+  
+  def test_format_3
+    d = create_driver(CONFIG + %[
+      out_key data
+    ])
+
+    d.run do
+      d.emit({"message" => "key1=val1 key2=val2"})
+    end
+
+    assert_equal [
+      {"data" => {"key1" => "val1", "key2" => "val2"}},
+    ], d.records
+  end
+
+  def test_format_keysprefix
+    d = create_driver(CONFIG + %[
+      out_key data
+      keys_prefix extracted_
+    ])
+
+    d.run do
+      d.emit({"message" => "key1=val1 key2=val2"})
+    end
+
+    assert_equal [
+      {"data" => {"extracted_key1" => "val1", "extracted_key2" => "val2"}},
+    ], d.records
+  end
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-split_record
+version: !ruby/object:Gem::Version
+  version: 0.12.1
+platform: ruby
+authors:
+- Masahiro Sano
+- Michael Adams
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-08-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.39
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.39
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+description: Fluentd filter plugin to split a record into multiple records with key/value
+  pair.
+email:
+- sabottenda@gmail.com
+- unquietwiki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-split_record.gemspec
+- lib/fluent/plugin/filter_split_record.rb
+- test/helper.rb
+- test/plugin/test_filter_split_record.rb
+homepage: https://github.com/unquietwiki/fluent-plugin-split_record
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Successor to original fluent-plugin-split. Updated for fluentd 0.12 and 0.14,
+  with non-conflicting name.
+test_files: []