fluent-plugin-s3 0.3.1 → 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +15 -0
- data/README.rdoc +12 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_s3.rb +20 -1
- data/test/out_s3.rb +11 -0
- metadata +4 -4
data/ChangeLog
CHANGED
@@ -1,3 +1,18 @@
|
|
1
|
+
Release 0.3.3 - 2013/06/18
|
2
|
+
|
3
|
+
* Fix require bug on case-sensitive environment
|
4
|
+
|
5
|
+
|
6
|
+
Release 0.3.2 - 2013/06/18
|
7
|
+
|
8
|
+
* Support lzo mime-type
|
9
|
+
https://github.com/fluent/fluent-plugin-s3/pull/29
|
10
|
+
* Add proxy_uri option
|
11
|
+
https://github.com/fluent/fluent-plugin-s3/issues/25
|
12
|
+
* Add check_apikey_on_start option
|
13
|
+
https://github.com/fluent/fluent-plugin-s3/pull/28
|
14
|
+
|
15
|
+
|
1
16
|
Release 0.3.1 - 2013/03/28
|
2
17
|
|
3
18
|
* Support json and text mime-types
|
data/README.rdoc
CHANGED
@@ -37,7 +37,7 @@ Simply use RubyGems:
|
|
37
37
|
|
38
38
|
[s3_bucket (required)] S3 bucket name.
|
39
39
|
|
40
|
-
[s3_endpoint] s3 endpoint name.
|
40
|
+
[s3_endpoint] s3 endpoint name. For example, US West (Oregon) Region is "s3-us-west-2.amazonaws.com". The full list of endpoints are available here. > http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
|
41
41
|
|
42
42
|
[s3_object_key_format] The format of S3 object keys. You can use several built-in variables:
|
43
43
|
|
@@ -79,8 +79,19 @@ The {fluent-mixin-config-placeholders}[https://github.com/tagomoris/fluent-mixin
|
|
79
79
|
|
80
80
|
s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
|
81
81
|
|
82
|
+
[store_as] archive format on S3. You can use serveral format:
|
83
|
+
|
84
|
+
- gzip (default)
|
85
|
+
- json
|
86
|
+
- text
|
87
|
+
- lzo (Need lzop command)
|
88
|
+
|
82
89
|
[auto_create_bucket] Create S3 bucket if it does not exists. Default is true.
|
83
90
|
|
91
|
+
[check_apikey_on_start] Check AWS key on start. Default is true.
|
92
|
+
|
93
|
+
[proxy_uri] uri of proxy environment.
|
94
|
+
|
84
95
|
[path] path prefix of the files on S3. Default is "" (no prefix).
|
85
96
|
|
86
97
|
[buffer_path (required)] path prefix of the files to buffer logs.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.3
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -11,6 +11,7 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
11
11
|
require 'zlib'
|
12
12
|
require 'time'
|
13
13
|
require 'tempfile'
|
14
|
+
require 'open3'
|
14
15
|
|
15
16
|
@use_ssl = true
|
16
17
|
end
|
@@ -31,6 +32,8 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
31
32
|
config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
32
33
|
config_param :store_as, :string, :default => "gzip"
|
33
34
|
config_param :auto_create_bucket, :bool, :default => true
|
35
|
+
config_param :check_apikey_on_start, :bool, :default => true
|
36
|
+
config_param :proxy_uri, :string, :default => nil
|
34
37
|
|
35
38
|
attr_reader :bucket
|
36
39
|
|
@@ -62,6 +65,13 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
62
65
|
|
63
66
|
@ext, @mime_type = case @store_as
|
64
67
|
when 'gzip' then ['gz', 'application/x-gzip']
|
68
|
+
when 'lzo' then
|
69
|
+
begin
|
70
|
+
Open3.capture3('lzop -V')
|
71
|
+
rescue Errno::ENOENT
|
72
|
+
raise ConfigError, "'lzop' utility must be in PATH for LZO compression"
|
73
|
+
end
|
74
|
+
['lzo', 'application/x-lzop']
|
65
75
|
when 'json' then ['json', 'application/json']
|
66
76
|
else ['txt', 'text/plain']
|
67
77
|
end
|
@@ -77,13 +87,14 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
77
87
|
options[:secret_access_key] = @aws_sec_key
|
78
88
|
end
|
79
89
|
options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
|
90
|
+
options[:proxy_uri] = @proxy_uri if @proxy_uri
|
80
91
|
options[:use_ssl] = @use_ssl
|
81
92
|
|
82
93
|
@s3 = AWS::S3.new(options)
|
83
94
|
@bucket = @s3.buckets[@s3_bucket]
|
84
95
|
|
85
96
|
ensure_bucket
|
86
|
-
check_apikeys
|
97
|
+
check_apikeys if @check_apikey_on_start
|
87
98
|
end
|
88
99
|
|
89
100
|
def format(tag, time, record)
|
@@ -128,6 +139,13 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
128
139
|
w = Zlib::GzipWriter.new(tmp)
|
129
140
|
chunk.write_to(w)
|
130
141
|
w.close
|
142
|
+
elsif @store_as == "lzo"
|
143
|
+
w = Tempfile.new("chunk-tmp")
|
144
|
+
chunk.write_to(w)
|
145
|
+
w.close
|
146
|
+
tmp.close
|
147
|
+
# We don't check the return code because we can't recover lzop failure.
|
148
|
+
system "lzop -qf1 -o #{tmp.path} #{w.path}"
|
131
149
|
else
|
132
150
|
chunk.write_to(tmp)
|
133
151
|
tmp.close
|
@@ -136,6 +154,7 @@ class S3Output < Fluent::TimeSlicedOutput
|
|
136
154
|
ensure
|
137
155
|
tmp.close(true) rescue nil
|
138
156
|
w.close rescue nil
|
157
|
+
w.unlink rescue nil
|
139
158
|
end
|
140
159
|
end
|
141
160
|
|
data/test/out_s3.rb
CHANGED
@@ -59,6 +59,17 @@ class S3OutputTest < Test::Unit::TestCase
|
|
59
59
|
assert_equal 'text/plain', d.instance.instance_variable_get(:@mime_type)
|
60
60
|
end
|
61
61
|
|
62
|
+
def test_configure_with_mime_type_lzo
|
63
|
+
conf = CONFIG.clone
|
64
|
+
conf << "\nstore_as lzo\n"
|
65
|
+
d = create_driver(conf)
|
66
|
+
assert_equal 'lzo', d.instance.instance_variable_get(:@ext)
|
67
|
+
assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@mime_type)
|
68
|
+
rescue => e
|
69
|
+
# TODO: replace code with disable lzop command
|
70
|
+
assert(e.is_a?(Fluent::ConfigError))
|
71
|
+
end
|
72
|
+
|
62
73
|
def test_format
|
63
74
|
d = create_driver
|
64
75
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-06-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -136,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
136
136
|
version: '0'
|
137
137
|
segments:
|
138
138
|
- 0
|
139
|
-
hash: -
|
139
|
+
hash: -3746525337841491520
|
140
140
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
141
141
|
none: false
|
142
142
|
requirements:
|
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
145
145
|
version: '0'
|
146
146
|
segments:
|
147
147
|
- 0
|
148
|
-
hash: -
|
148
|
+
hash: -3746525337841491520
|
149
149
|
requirements: []
|
150
150
|
rubyforge_project:
|
151
151
|
rubygems_version: 1.8.23
|