fluent-plugin-s3 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/ChangeLog CHANGED
@@ -1,4 +1,9 @@
1
1
 
2
+ Release 0.2.5 - 2012/12/06
3
+
4
+ * Add format_json and time/tag mixin options [#9]
5
+
6
+
2
7
  Release 0.2.4 - 2012/11/21
3
8
 
4
9
  * Set content type when writing file to s3
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.4
1
+ 0.2.5
@@ -15,6 +15,12 @@ class S3Output < Fluent::TimeSlicedOutput
15
15
  config_param :path, :string, :default => ""
16
16
  config_param :time_format, :string, :default => nil
17
17
 
18
+ include SetTagKeyMixin
19
+ config_set_default :include_tag_key, false
20
+
21
+ include SetTimeKeyMixin
22
+ config_set_default :include_time_key, false
23
+
18
24
  config_param :aws_key_id, :string, :default => nil
19
25
  config_param :aws_sec_key, :string, :default => nil
20
26
  config_param :s3_bucket, :string
@@ -23,6 +29,12 @@ class S3Output < Fluent::TimeSlicedOutput
23
29
  def configure(conf)
24
30
  super
25
31
 
32
+ if format_json = conf['format_json']
33
+ @format_json = true
34
+ else
35
+ @format_json = false
36
+ end
37
+
26
38
  @timef = TimeFormatter.new(@time_format, @localtime)
27
39
  end
28
40
 
@@ -39,8 +51,23 @@ class S3Output < Fluent::TimeSlicedOutput
39
51
  end
40
52
 
41
53
  def format(tag, time, record)
42
- time_str = @timef.format(time)
43
- "#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
54
+ if @include_time_key || !@format_json
55
+ time_str = @timef.format(time)
56
+ end
57
+
58
+ # copied from each mixin because current TimeSlicedOutput can't support mixins.
59
+ if @include_tag_key
60
+ record[@tag_key] = tag
61
+ end
62
+ if @include_time_key
63
+ record[@time_key] = time_str
64
+ end
65
+
66
+ if @format_json
67
+ Yajl.dump(record) + "\n"
68
+ else
69
+ "#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
70
+ end
44
71
  end
45
72
 
46
73
  def write(chunk)
data/test/out_s3.rb CHANGED
@@ -44,6 +44,76 @@ class S3OutputTest < Test::Unit::TestCase
44
44
  d.run
45
45
  end
46
46
 
47
+ def test_format_included_tag_and_time
48
+ config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
49
+ d = create_driver(config)
50
+
51
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
52
+ d.emit({"a"=>1}, time)
53
+ d.emit({"a"=>2}, time)
54
+
55
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
56
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
57
+
58
+ d.run
59
+ end
60
+
61
+ def test_format_with_format_json
62
+ config = [CONFIG, 'format_json true'].join("\n")
63
+ d = create_driver(config)
64
+
65
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
66
+ d.emit({"a"=>1}, time)
67
+ d.emit({"a"=>2}, time)
68
+
69
+ d.expect_format %[{"a":1}\n]
70
+ d.expect_format %[{"a":2}\n]
71
+
72
+ d.run
73
+ end
74
+
75
+ def test_format_with_format_json_included_tag
76
+ config = [CONFIG, 'format_json true', 'include_tag_key true'].join("\n")
77
+ d = create_driver(config)
78
+
79
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
80
+ d.emit({"a"=>1}, time)
81
+ d.emit({"a"=>2}, time)
82
+
83
+ d.expect_format %[{"a":1,"tag":"test"}\n]
84
+ d.expect_format %[{"a":2,"tag":"test"}\n]
85
+
86
+ d.run
87
+ end
88
+
89
+ def test_format_with_format_json_included_time
90
+ config = [CONFIG, 'format_json true', 'include_time_key true'].join("\n")
91
+ d = create_driver(config)
92
+
93
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
94
+ d.emit({"a"=>1}, time)
95
+ d.emit({"a"=>2}, time)
96
+
97
+ d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
98
+ d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
99
+
100
+ d.run
101
+ end
102
+
103
+ def test_format_with_format_json_included_tag_and_time
104
+ config = [CONFIG, 'format_json true', 'include_tag_key true', 'include_time_key true'].join("\n")
105
+ d = create_driver(config)
106
+
107
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
108
+ d.emit({"a"=>1}, time)
109
+ d.emit({"a"=>2}, time)
110
+
111
+ d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
112
+ d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
113
+
114
+ d.run
115
+ end
116
+
47
117
  def test_write
48
118
  d = create_driver
49
119
 
@@ -55,7 +125,7 @@ class S3OutputTest < Test::Unit::TestCase
55
125
  data = d.run
56
126
 
57
127
  assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
58
- %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
128
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
59
129
  data
60
130
  end
61
131
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.4
4
+ version: 0.2.5
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-11-21 00:00:00.000000000 Z
12
+ date: 2012-12-06 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd