fluent-plugin-s3 0.2.4 → 0.2.5

data/ChangeLog

@@ -1,4 +1,9 @@
 
+Release 0.2.5 - 2012/12/06
+
+  * Add format_json and time/tag mixin options [#9]
+
+
 Release 0.2.4 - 2012/11/21
 
   * Set content type when writing file to s3

data/VERSION

@@ -1 +1 @@
-0.2.4
+0.2.5

data/lib/fluent/plugin/out_s3.rb

@@ -15,6 +15,12 @@ class S3Output < Fluent::TimeSlicedOutput
   config_param :path, :string, :default => ""
   config_param :time_format, :string, :default => nil
 
+  include SetTagKeyMixin
+  config_set_default :include_tag_key, false
+
+  include SetTimeKeyMixin
+  config_set_default :include_time_key, false
+
   config_param :aws_key_id, :string, :default => nil
   config_param :aws_sec_key, :string, :default => nil
   config_param :s3_bucket, :string
@@ -23,6 +29,12 @@ class S3Output < Fluent::TimeSlicedOutput
   def configure(conf)
     super
 
+    if format_json = conf['format_json']
+      @format_json = true
+    else
+      @format_json = false
+    end
+
     @timef = TimeFormatter.new(@time_format, @localtime)
   end
 
@@ -39,8 +51,23 @@ class S3Output < Fluent::TimeSlicedOutput
   end
 
   def format(tag, time, record)
-    time_str = @timef.format(time)
-    "#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
+    if @include_time_key || !@format_json
+      time_str = @timef.format(time)
+    end
+
+    # copied from each mixin because current TimeSlicedOutput can't support mixins.
+    if @include_tag_key
+      record[@tag_key] = tag
+    end
+    if @include_time_key
+      record[@time_key] = time_str
+    end
+
+    if @format_json
+      Yajl.dump(record) + "\n"
+    else
+      "#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
+    end
   end
 
   def write(chunk)

data/test/out_s3.rb

@@ -44,6 +44,76 @@ class S3OutputTest < Test::Unit::TestCase
     d.run
   end
 
+  def test_format_included_tag_and_time
+    config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json
+    config = [CONFIG, 'format_json true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1}\n]
+    d.expect_format %[{"a":2}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json_included_tag
+    config = [CONFIG, 'format_json true', 'include_tag_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1,"tag":"test"}\n]
+    d.expect_format %[{"a":2,"tag":"test"}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json_included_time
+    config = [CONFIG, 'format_json true', 'include_time_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
+    d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
+
+    d.run
+  end
+
+  def test_format_with_format_json_included_tag_and_time
+    config = [CONFIG, 'format_json true', 'include_tag_key true', 'include_time_key true'].join("\n")
+    d = create_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+    d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+
+    d.run
+  end
+
   def test_write
     d = create_driver
 
@@ -55,7 +125,7 @@ class S3OutputTest < Test::Unit::TestCase
     data = d.run
 
     assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
-                    %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                 %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
                  data
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-21 00:00:00.000000000 Z
+date: 2012-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd