fluent-plugin-kv-parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 787237ea7e0cdfbefdef04d5190387575c51ebd9
+  data.tar.gz: f20c58f82c89bbcd528757dbd82806610182df83
+SHA512:
+  metadata.gz: 40a7a05d055b54ef13fe08ad99bcf22bedc74970b65316659a39c8d677c5629f1985c514f39dcf1285d696ef1f0b38a916d4165ed53cd3d87cca12255d00adf0
+  data.tar.gz: 56f423018fb7e35d05ec54acd524d5f33fccb25ef5f09a5f58c00f83189992eb30cfc3b3167349eaa23fb52f9e79152c025f75e24b473ca64f5187e1a450dd57

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,78 @@
+# Key-Value Pairs Parser Plugin for [Fluentd](https://github.com/fluent/fluentd)
+
+## Overview
+
+This is a parser plugin for Fluentd. Learn more about parser plugins [here](https://docs.fluentd.org/articles/parser-plugin-overview).
+
+This plugin allows you to parse inputs that look like key-value pairs. For example, if your text logs look like
+
+```
+"this_field=10000  that_field=hello time=2013-01-01T12:34:00"
+```
+
+It is parsed as
+
+```json
+{"this_field":10000, "that_field":"hello"}
+```
+
+with the event's time being `2013-01-01T12:34:00`
+
+## How to Install and Use
+
+For Fluentd,
+
+```
+gem install fluent-plugin-kv-parser
+```
+
+For [Treasure Agent](https://docs.treasuredata.com/articles/td-agent),
+
+```
+/usr/sbin/td-agent-gem install fluent-plugin-kv-parser
+```
+
+Then, for parser-plugin enabled input plugins (including [in_tail](https://docs.fluentd.org/articles/in_tail), [in_tcp](https://docs.fluentd.org/articles/in_tcp), [in_udp](https://docs.fluentd.org/articles/in_udp) and [in_syslog](https://docs.fluentd.org/articles/syslog)), you can just write `format kv`
+
+For example, using `in_tcp` with the following configuration:
+
+```aconf
+<source>
+  type tcp
+  port 24225
+  tag kv_log
+  format kv
+  time_key my_time
+  types k1:integer,my_time:time
+</source>
+<match kv_log>
+  type stdout
+</match>
+```
+
+Running
+
+```shell
+echo 'my_time=2014-12-31T00:00:00 k1=1234 k2=hello' | nc localhost 24224
+```
+
+gives
+
+```shell
+2014-12-31 00:00:00 +0000 kv_log: {"k1":1234,"k2":"hello"}
+```
+
+## Parameters
+
+* **kv_delimiter**: The delimiter for key-value pairs. By default, it is the regexp `/\t\s/+` (one or more whitespace/tabs). If the value starts and ends with the character `'/'`, the separator is interpreted to be a regexp. Else, it is interpreted to be a string. Hence,
+    
+    - `kv_delimiter /a+/` splits on one or more "a"s
+    - `kv_delimiter a` splits on a single "a"
+
+* **kv_char**: The string to split the key from the value. By default, it is "=".
+* **time_key**: The time key field among the key-value pairs to be used as the time for the event. If missing or unparsable, the current time is used.
+* **types**: The parameter to convert the values of key-value pairs. The syntax is `<key_name>:<type_name>`. For example, to convert the key "k1" into integer, write `types k1:integer`. For the `time` type, one can write `<key_name>:time:<time_format>` to convert the string into a time object. For example, to convert the string "my_time=12/31/2014 12:00:00", use `my_time:time:%m/%d/%Y %H:%M:%S`. This parameter is same as the one used for [in_tail](https://docs.fluentd.org/articles/in_tail) and others (see under the "types" section over there).
+
+## License
+
+Apache 2.0. Copyright Kiyoto Tamura

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << "test" << "lib"
+  test.pattern = 'test/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-docker-metrics.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-kv-parser"
+  spec.version       = "0.0.1"
+  spec.description   = 'Fluentd parser plugin to parse key value pairs'
+  spec.authors       = ["kiyoto"]
+  spec.email         = ["kiyoto@treasure-data.com"]
+  spec.summary       = %q{Fluentd parser plugin to parse key value pairs}
+  spec.homepage      = "https://github.com/kiyoto/fluent-plugin-kv-parser"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", '~> 1.7'
+  spec.add_development_dependency "rake", '~> 10.1'
+  spec.add_runtime_dependency "fluentd", '~> 0.10'
+end

data/lib/fluent/plugin/parser_kv.rb

@@ -0,0 +1,51 @@
+module Fluent
+  class TextParser
+    class KVParser < Parser
+      include Configurable
+      include TypeConverter
+
+      config_param :kv_delimiter, :string, :default => '/[\t\s]+/'
+      config_param :kv_char, :string, :default => '='
+      config_param :time_key, :string, :default => 'time'
+
+      def configure(conf={})
+        super
+        if @kv_delimiter[0] == '/' and @kv_delimiter[-1] == '/'
+          @kv_delimiter = Regexp.new(@kv_delimiter[1..-2])
+        end
+      end
+
+      def parse(text)
+        record = {}
+        text.split(@kv_delimiter).each do |kv|
+          k, v = kv.split(@kv_char, 2)
+          record[k] = v
+        end
+
+        convert_field_type!(record) if @type_converters
+        time = record.delete(@time_key)
+        if time.nil?
+          time = Engine.now
+        elsif time.respond_to?(:to_i)
+          time = time.to_i
+        else
+          raise RuntimeError, "The #{@time_key}=#{time} is a bad time field"
+        end
+
+        yield time, record
+      end
+
+      private
+
+      def convert_field_type!(record)
+        @type_converters.each_key { |key|
+          if value = record[key]
+            record[key] = convert_type(key, value)
+          end
+        }
+      end
+
+    end
+    register_template('kv', Proc.new { KVParser.new })
+  end
+end

data/test/test_kv_parser.rb

@@ -0,0 +1,57 @@
+require 'fluent/test'
+require 'fluent/parser'
+require 'fluent/plugin/parser_kv'
+
+module ParserTest
+  include Fluent
+
+  class KVParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def test_basic
+      parser = Fluent::TextParser::KVParser.new
+      parser.configure
+      parser.parse("k1=v1 k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+      parser.parse("k1=v1    k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+      parser.parse("k2=v2 k1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+      parser.parse("k2=v2\tk1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+      parser.parse("k2=v2\t k1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+    end
+
+    def test_with_types
+      parser = Fluent::TextParser::KVParser.new
+      parser.configure("types" => "k1:integer")
+      parser.parse("k1=100") {|_, v| assert_equal(100, v["k1"])}
+    end
+
+    def test_with_time
+      parser = Fluent::TextParser::KVParser.new
+      parser.configure("types" => "time:time:%Y-%m-%dT%H:%M:%S")
+      parser.parse("k1=foo time=1970-01-01T01:00:00") {|time, v|
+        assert_equal(3600, time)
+        assert_equal("foo", v["k1"])
+      }
+    end
+
+    def test_custom_delimiter
+      parser = Fluent::TextParser::KVParser.new
+      parser.configure("kv_delimiter" => "|")
+      parser.parse("k1=v1|k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+      parser.configure("kv_delimiter" => "/[@ ]/")
+      parser.parse("k1=v1@k2=v2 k3=v3") {|_, v|
+        assert_equal({"k1"=>"v1", "k2"=>"v2", "k3"=>"v3"}, v)
+      }
+    end
+
+    def test_custom_kv_char
+      parser = Fluent::TextParser::KVParser.new
+      parser.configure("kv_char" => "#")
+      parser.parse("k1#v1 k2#v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+    end
+
+    def test_types_param
+      parser = Fluent::TextParser::KVParser.new
+    end
+
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-kv-parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- kiyoto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.1'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: Fluentd parser plugin to parse key value pairs
+email:
+- kiyoto@treasure-data.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-docker-metrics.gemspec
+- lib/fluent/plugin/parser_kv.rb
+- test/test_kv_parser.rb
+homepage: https://github.com/kiyoto/fluent-plugin-kv-parser
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Fluentd parser plugin to parse key value pairs
+test_files:
+- test/test_kv_parser.rb