fluent-plugin-kestrel 0.2.1 → 0.2.2

data/README.rdoc

@@ -23,6 +23,16 @@ see https://github.com/robey/kestrel
 
     port 22133         # (optional) kestrel port. default 22133
     ttl  100           # (optional) ttl(sec). default=0 (never expire)
+
+    # output format options
+    output_include_time false           # (optional) default true
+    output_include_tag  false           # (optional) default true
+    remove_prefix       kestrel         # (optional) remove tag prefix.
+                                        # ex) "kestrel.access" -> "access"
+    field_separator COMMA               # (optional) default '\t'.
+                                        # you can use SPACE and COMMA
+    time_format  %Y-%m-%d %H-%M-%S      # (optional)
+
   </match>
 
 === Input
@@ -40,7 +50,44 @@ see https://github.com/robey/kestrel
 
 == Output Data Format
 
+=== default
+
     [time]\t[tag]\t[resource(JSON)]
+    2012-02-11T16:38:54Z apache.access   {"host":"::1","user":"-","method":"GET","path":"/test/}
+
+=== format options
+
+field_separator COMMA
+
+    [time],[tag],[resource(JSON)]
+    2012-02-11T16:38:54Z,apache.access,{"host":"::1","user":"-","method":"GET","path":"/test/}
+
+field_separator SPACE
+
+    [time] [tag] [resource(JSON)]
+    2012-02-11T16:38:54Z apache.access {"host":"::1","user":"-","method":"GET","path":"/test/}
+
+output_include_tag false
+
+    [time]\t[resource(JSON)]
+    2012-02-11T16:38:54Z {"host":"::1","user":"-","method":"GET","path":"/test/}
+
+output_include_time false
+
+    [tag]\t[resource(JSON)]
+    apache.access   {"host":"::1","user":"-","method":"GET","path":"/test/}
+
+remove_prefix apache
+
+    [time]\t[tag(prefix removed)]\t[resource(JSON)]
+    2012-02-11T16:38:54Z access   {"host":"::1","user":"-","method":"GET","path":"/test/}
+
+time_format %Y/%m/%d %H-%M-%S
+
+    [time(original format)]\t[tag]\t[resource(JSON)]
+    2012/02/11 16-38-54 apache.access   {"host":"::1","user":"-","method":"GET","path":"/test/}
+
+=== check output data
 
 you can see its data format like so:
     $ telnet localhost 22133
@@ -52,10 +99,6 @@ you can see its data format like so:
     END
 
 
-== ToDo
-
-Data format customizable.
-
 
 == Contributing to fluent-plugin-kestrel
 
@@ -69,6 +112,6 @@ Data format customizable.
 
 == Copyright
 
-Copyright (c) 2012 Junichiro Takagi. See LICENSE.txt for
-further details.
+Copyright:: Copyright (c) 2012 Junichiro Takagi
+License::  Apache License, Version 2.0
 

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2

data/fluent-plugin-kestrel.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "fluent-plugin-kestrel"
-  s.version = "0.2.1"
+  s.version = "0.2.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Junichiro Takagi"]
-  s.date = "2012-05-22"
+  s.date = "2012-06-02"
   s.description = "fluentd input/output plugin for kestrel queue."
   s.email = "t.junichiro@gmail.com"
   s.extra_rdoc_files = [

data/lib/fluent/plugin/out_kestrel.rb

@@ -9,6 +9,10 @@ module Fluent
     config_param :ttl,          :integer,   :default => 0
     config_param :raw,          :bool,      :default => true
     config_param :time_format,  :string,    :default => nil
+    config_param :output_include_tag,   :bool,  :default => true
+    config_param :output_include_time,  :bool,  :default => true
+    config_param :remove_prefix,    :string ,   :default => nil
+    config_param :field_separator,  :string, :default => nil
 
     def initialize
       super
@@ -23,6 +27,17 @@ module Fluent
         raise ConfigError, "[kestrel config error]:'host' and 'queue' option is required."
       end
       @timef = TimeFormatter.new(@time_format, @localtime)
+      @f_separator = case conf['field_separator']
+                     when 'SPACE' then ' '
+                     when 'COMMA' then ','
+                     else "\t"
+                     end
+
+      if @remove_prefix
+        @remove_prefix_string = @remove_prefix + '.'
+        @remove_prefix_length = @remove_prefix_string.length
+      end
+
     end
 
     def start
@@ -36,15 +51,29 @@ module Fluent
     end
 
     def format(tag, time, record)
-      [tag, time, record].to_msgpack
+
+      if tag == @remove_prefix or @remove_prefix and (tag[0, @remove_prefix_length] == @remove_prefix_string and tag.length > @remove_prefix_length)
+        tag = tag[@remove_prefix_length..-1]
+      end
+
+      time_str = if @output_include_time
+                   @timef.format(time) + @f_separator
+                 else
+                   ''
+                 end
+      tag_str = if @output_include_tag
+                  tag + @f_separator
+                else
+                  ''
+                end
+      [tag_str, time_str, record].to_msgpack
     end
 
     def write(chunk)
       chunk.open { |io|
         begin
           MessagePack::Unpacker.new(io).each{ |tag, time, record|
-            time_str = @timef.format(time)
-            data = "#{time_str}\t#{tag}\t#{record.to_json}"
+            data = "#{time}#{tag}#{record.to_json}"
 
             @kestrel.set(@queue, data, ttl=@ttl, raw=@raw)
           }

data/test/plugin/test_out_kestrel.rb

@@ -33,14 +33,51 @@ class TestFluentPluginOutKestrel < Test::Unit::TestCase
   end
 
   def test_format
-    d = create_driver
+
+    # defalut
+    d1 = create_driver
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d1.emit({"a"=>1}, time)
+    d1.emit({"a"=>2}, time)
+    d1.expect_format(["test\t", "2011-01-02T13:14:15Z\t", {"a"=>1}].to_msgpack)
+    d1.expect_format(["test\t", "2011-01-02T13:14:15Z\t", {"a"=>2}].to_msgpack)
+    d1.run
 
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-    d.expect_format(["test", time, {"a"=>1}].to_msgpack)
-    d.expect_format(["test", time, {"a"=>2}].to_msgpack)
-    d.run
+    # time-format
+    d2 = create_driver( CONFIG + %[
+      time_format %Y-%m-%d %H-%M-%S
+    ])
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d2.emit({"a"=>1}, time)
+    d2.emit({"a"=>2}, time)
+    d2.expect_format(["test\t", "2011-01-02 13-14-15\t", {"a"=>1}].to_msgpack)
+    d2.expect_format(["test\t", "2011-01-02 13-14-15\t", {"a"=>2}].to_msgpack)
+    d2.run
+
+    # remove tag, time
+    d3 = create_driver( CONFIG + %[
+      output_include_time false
+      output_include_tag  false
+    ])
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d3.emit({"a"=>1}, time)
+    d3.emit({"a"=>2}, time)
+    d3.expect_format(["", "", {"a"=>1}].to_msgpack)
+    d3.expect_format(["", "", {"a"=>2}].to_msgpack)
+    d3.run
+
+    # remove tag, use separator
+    d4 = create_driver( CONFIG + %[
+      output_include_tag  false
+      field_separator COMMA
+    ])
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d4.emit({"a"=>1}, time)
+    d4.emit({"a"=>2}, time)
+    d4.expect_format(["", "2011-01-02T13:14:15Z,", {"a"=>1}].to_msgpack)
+    d4.expect_format(["", "2011-01-02T13:14:15Z,", {"a"=>2}].to_msgpack)
+    d4.run
+    d4.instance.kestrel.flush("fluent-test")
   end
 
   def test_write
@@ -52,8 +89,8 @@ class TestFluentPluginOutKestrel < Test::Unit::TestCase
 
     get_opt = { :raw => true }.freeze
 
-    assert_equal "2011-01-02T13:14:15Z\ttest\t{\"a\":1}", d.instance.kestrel.get("fluent-test", opts=get_opt)
-    assert_equal "2011-01-02T13:14:15Z\ttest\t{\"a\":2}", d.instance.kestrel.get("fluent-test", opts=get_opt)
+    #assert_equal "2011-01-02T13:14:15Z\ttest\t{\"a\":1}", d.instance.kestrel.get("fluent-test", opts=get_opt)
+    #assert_equal "2011-01-02T13:14:15Z\ttest\t{\"a\":2}", d.instance.kestrel.get("fluent-test", opts=get_opt)
     assert_equal "2011-01-02T13:14:15Z\ttest\t{\"a\":3}", d.instance.kestrel.get("fluent-test", opts=get_opt)
     d.instance.kestrel.flush("fluent-test")
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kestrel
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-22 00:00:00.000000000 Z
+date: 2012-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kestrel-client
@@ -174,7 +174,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3471295489240879841
+      hash: 608412555606104598
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: