fluent-plugin-kv-parser 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fe95909c734f6a521e7ee45bcc325afe33649f8f
-  data.tar.gz: a10bd9c7249327b77f91222c60d69444ba0ef0da
+SHA256:
+  metadata.gz: f4b7b45b73282ce3360af992bf3f56eab1b6a043ed25d8ee37ffad639079a61f
+  data.tar.gz: 63d383d9edc7cf3d46f7d9176e3b7742ca3e26cae50854c27693528d8243a44e
 SHA512:
-  metadata.gz: 44d02fcdf9fb09e851daa6f3fa7432c6e528c80ecd3f07ae0eefe27336732340868b34ab69f54418a3a83fc16e20a77d3b3cb35755b2936f23ded15aa3269a87
-  data.tar.gz: a7d7c9bb816335ebd0ce5d207f9bc662f38a21e676685adeb5047faba07473dc6754e1c2b0778e22a6507f9bba2a34dc0690fab7a99f36bf8145ac9fba850c60
+  metadata.gz: 52ae18e5a422586ca55fb78eb3598d9f880a111e22ced3a56e743a55e1a8584ee37d3c374cc8beef631b619f846e0ee23b75f99ba614742c7054d46a9d8565f4
+  data.tar.gz: 8b73739ebc3ece980df58d9836e15730c8fd1a0ebdfd52f3174ce7a1d404855d222483684a5b906ae45aa852a94cfc30848f465896087c2e235adaf2b660e765

data/.travis.yml

@@ -1,12 +1,11 @@
 language: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
   - 2.1
   - 2.2
   - 2.3
-  - 2.4
+  - 2.4.3
+  - 2.5.0
   - ruby-head
   - rbx-2
 

data/README.md

@@ -1,6 +1,6 @@
 # Key-Value Pairs Parser Plugin for [Fluentd](https://github.com/fluent/fluentd)
 
-<img src="https://travis-ci.org/kiyoto/fluent-plugin-kv-parser.svg?branch=master"/>
+[![Build Status](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-kv-parser.svg?branch=master)](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-kv-parser)
 
 ## Overview
 
@@ -20,6 +20,13 @@ It is parsed as
 
 with the event's time being `2013-01-01T12:34:00`
 
+## Requirements
+
+| fluent-plugin-kv-parser | fluentd    | ruby   |
+|-------------------------|------------|--------|
+| >= 1.0.0                | >= v0.14.0 | >= 2.1 |
+| <  1.0.0                | >= v0.12.0 | >= 1.9 |
+
 ## How to Install and Use
 
 For Fluentd,
@@ -40,15 +47,17 @@ For example, using `in_tcp` with the following configuration:
 
 ```aconf
 <source>
-  type tcp
+  @type tcp
   port 24225
   tag kv_log
-  format kv
-  time_key my_time
-  types k1:integer,my_time:time
+  <parse>
+    @type kv
+    time_key my_time
+    types k1:integer,my_time:time
+  </parse>
 </source>
 <match kv_log>
-  type stdout
+  @type stdout
 </match>
 ```
 

data/fluent-plugin-kv-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-kv-parser"
-  spec.version       = "0.1.0"
+  spec.version       = "1.0.0"
   spec.description   = 'Fluentd parser plugin to parse key value pairs'
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
@@ -19,5 +19,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", "> 3.0"
-  spec.add_runtime_dependency "fluentd", '~> 0.12.0'
+  spec.add_runtime_dependency "fluentd", '> 0.14.0', '< 2'
 end

data/lib/fluent/plugin/parser_kv.rb

@@ -1,14 +1,16 @@
+require "fluent/plugin/parser"
+
 module Fluent
-  class TextParser
-    class KVParser < Parser
-      include Configurable
-      include TypeConverter
+  module Plugin
+    class KVParser < Fluent::Plugin::Parser
+      Fluent::Plugin.register_parser("kv", self)
+
+      config_param :kv_delimiter, :string, default: '/\s+/'
+      config_param :kv_char, :string, default: '='
 
-      config_param :kv_delimiter, :string, :default => '/[\t\s]+/'
-      config_param :kv_char, :string, :default => '='
-      config_param :time_key, :string, :default => 'time'
+      config_set_default :time_key, "time"
 
-      def configure(conf={})
+      def configure(conf)
         super
         if @kv_delimiter[0] == '/' and @kv_delimiter[-1] == '/'
           @kv_delimiter = Regexp.new(@kv_delimiter[1..-2])
@@ -18,34 +20,15 @@ module Fluent
       def parse(text)
         record = {}
         text.split(@kv_delimiter).each do |kv|
-          k, v = kv.split(@kv_char, 2)
-          record[k] = v
+          key, value = kv.split(@kv_char, 2)
+          record[key] = value
         end
 
-        convert_field_type!(record) if @type_converters
-        time = record.delete(@time_key)
-        if time.nil?
-          time = Engine.now
-        elsif time.respond_to?(:to_i)
-          time = time.to_i
-        else
-          raise RuntimeError, "The #{@time_key}=#{time} is a bad time field"
-        end
+        time = parse_time(record)
+        time, record = convert_values(time, record)
 
         yield time, record
       end
-
-      private
-
-      def convert_field_type!(record)
-        @type_converters.each_key { |key|
-          if value = record[key]
-            record[key] = convert_type(key, value)
-          end
-        }
-      end
-
     end
-    register_template('kv', Proc.new { KVParser.new })
   end
-end
+end

data/test/test_kv_parser.rb

@@ -1,73 +1,67 @@
 require 'fluent/test'
-require 'fluent/parser'
+require 'fluent/test/driver/parser'
+require 'fluent/test/helpers'
 require 'fluent/plugin/parser_kv'
 
 class KVParserTest < ::Test::Unit::TestCase
-
-  def create_driver(conf = {})
-    Fluent::Test::ParserTestDriver.new(Fluent::TextParser::KVParser).configure(conf)
-  end
+  include Fluent::Test::Helpers
 
   def setup
-    @timezone = ENV["TZ"]
-    ENV["TZ"] = "UTC"
+    Fluent::Test.setup
   end
 
-  def teardown
-    ENV["TZ"] = @timezone
+  def create_driver(conf={})
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::KVParser).configure(conf)
   end
 
-  def test_basic
+  data("single space" => ["k1=v1 k2=v2", { "k1" => "v1", "k2" => "v2" }],
+       "multiple space" => ["k1=v1    k2=v2", { "k1" => "v1", "k2" => "v2" }],
+       "reverse" => ["k2=v2 k1=v1", { "k1" => "v1", "k2" => "v2" }],
+       "tab" => ["k2=v2\tk1=v1", { "k1" => "v1", "k2" => "v2" }],
+       "tab and space" => ["k2=v2\t k1=v1", { "k1" => "v1", "k2" => "v2" }])
+  test "parse" do |(text, expected)|
     d = create_driver
-    parser = d.instance
-    parser.parse("k1=v1 k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
-    parser.parse("k1=v1    k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
-    parser.parse("k2=v2 k1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
-    parser.parse("k2=v2\tk1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
-    parser.parse("k2=v2\t k1=v1") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+    d.instance.parse(text) do |_time, record|
+      assert_equal(expected, record)
+    end
   end
 
-  def test_with_types
+  test "parse with types" do
     d = create_driver("types" => "k1:integer")
-    parser = d.instance
-    parser.parse("k1=100") {|_, v| assert_equal(100, v["k1"])}
+    d.instance.parse("k1=100") do |_time, record|
+      assert_equal({ "k1" => 100 }, record)
+    end
   end
 
-  def test_with_time
+  test "parse with time" do
     d = create_driver("types" => "time:time")
-    parser = d.instance
-    parser.parse("k1=foo time=1970-01-01T01:00:00") {|time, v|
-      assert_equal(3600, time)
-      assert_equal("foo", v["k1"])
-    }
+    d.instance.parse("k1=foo time=1970-01-01T01:00:00") do |time, record|
+      assert_equal(event_time("1970-01-01T01:00:0"), time)
+      assert_equal({ "k1" => "foo" }, record)
+    end
   end
 
-  def test_with_custom_time_key
+  test "parse with custom time_key" do
     d = create_driver("time_key" => "my_time", "types" => "my_time:time")
-    parser = d.instance
-    parser.parse("k1=foo my_time=1970-01-01T01:00:00") {|time, v|
-      assert_equal(3600, time)
-      assert_equal("foo", v["k1"])
-    }
-  end
-
-  def test_custom_delimiter
-    d = create_driver("kv_delimiter" => "|")
-    parser = d.instance
-    parser.parse("k1=v1|k2=v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+    d.instance.parse("k1=foo my_time=1970-01-01T01:00:00") do |time, record|
+      assert_equal(event_time("1970-01-01T01:00:0"), time)
+      assert_equal({ "k1" => "foo" }, record)
+    end
   end
 
-  def test_custom_delimiter2
-    d = create_driver("kv_delimiter" => "/[@ ]/")
-    parser = d.instance
-    parser.parse("k1=v1@k2=v2 k3=v3") {|_, v|
-      assert_equal({"k1"=>"v1", "k2"=>"v2", "k3"=>"v3"}, v)
-    }
+  data("pipe" => ["|", "k1=v1|k2=v2", {"k1" => "v1", "k2" => "v2" }],
+       "regexp" => ["/[@ ]/", "k1=v1@k2=v2 k3=v3", { "k1" => "v1", "k2" => "v2", "k3" => "v3" }])
+  test "parse with custom kv_delimiter" do |(delimiter, text, expected)|
+    d = create_driver("kv_delimiter" => delimiter)
+    d.instance.parse(text) do |_time, record|
+      assert_equal(expected, record)
+    end
   end
 
-  def test_custom_kv_char
+  test "parse with custom kv_char" do
     d = create_driver("kv_char" => "#")
-    parser = d.instance
-    parser.parse("k1#v1 k2#v2") {|_, v| assert_equal({"k1"=>"v1", "k2"=>"v2"}, v)}
+    d.instance.parse("k1#v1 k2#v2") do |_time, record|
+      assert_equal({ "k1" => "v1", "k2" => "v2" }, record)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kv-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - kiyoto
@@ -42,16 +42,22 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: '2'
 description: Fluentd parser plugin to parse key value pairs
 email:
 - kiyoto@treasure-data.com
@@ -86,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Fluentd parser plugin to parse key value pairs