fluent-plugin-s3 0.2.4 → 0.2.5
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +5 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_s3.rb +29 -2
- data/test/out_s3.rb +71 -1
- metadata +2 -2
data/ChangeLog
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.5
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -15,6 +15,12 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
15
15
|
config_param :path, :string, :default => ""
|
16
16
|
config_param :time_format, :string, :default => nil
|
17
17
|
|
18
|
+
include SetTagKeyMixin
|
19
|
+
config_set_default :include_tag_key, false
|
20
|
+
|
21
|
+
include SetTimeKeyMixin
|
22
|
+
config_set_default :include_time_key, false
|
23
|
+
|
18
24
|
config_param :aws_key_id, :string, :default => nil
|
19
25
|
config_param :aws_sec_key, :string, :default => nil
|
20
26
|
config_param :s3_bucket, :string
|
@@ -23,6 +29,12 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
23
29
|
def configure(conf)
|
24
30
|
super
|
25
31
|
|
32
|
+
if format_json = conf['format_json']
|
33
|
+
@format_json = true
|
34
|
+
else
|
35
|
+
@format_json = false
|
36
|
+
end
|
37
|
+
|
26
38
|
@timef = TimeFormatter.new(@time_format, @localtime)
|
27
39
|
end
|
28
40
|
|
@@ -39,8 +51,23 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
39
51
|
end
|
40
52
|
|
41
53
|
def format(tag, time, record)
|
42
|
-
|
43
|
-
|
54
|
+
if @include_time_key || !@format_json
|
55
|
+
time_str = @timef.format(time)
|
56
|
+
end
|
57
|
+
|
58
|
+
# copied from each mixin because current TimeSlicedOutput can't support mixins.
|
59
|
+
if @include_tag_key
|
60
|
+
record[@tag_key] = tag
|
61
|
+
end
|
62
|
+
if @include_time_key
|
63
|
+
record[@time_key] = time_str
|
64
|
+
end
|
65
|
+
|
66
|
+
if @format_json
|
67
|
+
Yajl.dump(record) + "\n"
|
68
|
+
else
|
69
|
+
"#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
|
70
|
+
end
|
44
71
|
end
|
45
72
|
|
46
73
|
def write(chunk)
|
data/test/out_s3.rb
CHANGED
@@ -44,6 +44,76 @@ class S3OutputTest < Test::Unit::TestCase
|
|
44
44
|
d.run
|
45
45
|
end
|
46
46
|
|
47
|
+
def test_format_included_tag_and_time
|
48
|
+
config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
|
49
|
+
d = create_driver(config)
|
50
|
+
|
51
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
52
|
+
d.emit({"a"=>1}, time)
|
53
|
+
d.emit({"a"=>2}, time)
|
54
|
+
|
55
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
56
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
57
|
+
|
58
|
+
d.run
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_format_with_format_json
|
62
|
+
config = [CONFIG, 'format_json true'].join("\n")
|
63
|
+
d = create_driver(config)
|
64
|
+
|
65
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
66
|
+
d.emit({"a"=>1}, time)
|
67
|
+
d.emit({"a"=>2}, time)
|
68
|
+
|
69
|
+
d.expect_format %[{"a":1}\n]
|
70
|
+
d.expect_format %[{"a":2}\n]
|
71
|
+
|
72
|
+
d.run
|
73
|
+
end
|
74
|
+
|
75
|
+
def test_format_with_format_json_included_tag
|
76
|
+
config = [CONFIG, 'format_json true', 'include_tag_key true'].join("\n")
|
77
|
+
d = create_driver(config)
|
78
|
+
|
79
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
80
|
+
d.emit({"a"=>1}, time)
|
81
|
+
d.emit({"a"=>2}, time)
|
82
|
+
|
83
|
+
d.expect_format %[{"a":1,"tag":"test"}\n]
|
84
|
+
d.expect_format %[{"a":2,"tag":"test"}\n]
|
85
|
+
|
86
|
+
d.run
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_format_with_format_json_included_time
|
90
|
+
config = [CONFIG, 'format_json true', 'include_time_key true'].join("\n")
|
91
|
+
d = create_driver(config)
|
92
|
+
|
93
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
94
|
+
d.emit({"a"=>1}, time)
|
95
|
+
d.emit({"a"=>2}, time)
|
96
|
+
|
97
|
+
d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
|
98
|
+
d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
|
99
|
+
|
100
|
+
d.run
|
101
|
+
end
|
102
|
+
|
103
|
+
def test_format_with_format_json_included_tag_and_time
|
104
|
+
config = [CONFIG, 'format_json true', 'include_tag_key true', 'include_time_key true'].join("\n")
|
105
|
+
d = create_driver(config)
|
106
|
+
|
107
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
108
|
+
d.emit({"a"=>1}, time)
|
109
|
+
d.emit({"a"=>2}, time)
|
110
|
+
|
111
|
+
d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
112
|
+
d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
113
|
+
|
114
|
+
d.run
|
115
|
+
end
|
116
|
+
|
47
117
|
def test_write
|
48
118
|
d = create_driver
|
49
119
|
|
@@ -55,7 +125,7 @@ class S3OutputTest < Test::Unit::TestCase
|
|
55
125
|
data = d.run
|
56
126
|
|
57
127
|
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
58
|
-
|
128
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
59
129
|
data
|
60
130
|
end
|
61
131
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-12-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|