fluent-plugin-s3 0.3.1 → 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
data/ChangeLog CHANGED
@@ -1,3 +1,18 @@
1
+ Release 0.3.3 - 2013/06/18
2
+
3
+ * Fix require bug on case-sensitive environment
4
+
5
+
6
+ Release 0.3.2 - 2013/06/18
7
+
8
+ * Support lzo mime-type
9
+ https://github.com/fluent/fluent-plugin-s3/pull/29
10
+ * Add proxy_uri option
11
+ https://github.com/fluent/fluent-plugin-s3/issues/25
12
+ * Add check_apikey_on_start option
13
+ https://github.com/fluent/fluent-plugin-s3/pull/28
14
+
15
+
1
16
  Release 0.3.1 - 2013/03/28
2
17
 
3
18
  * Support json and text mime-types
data/README.rdoc CHANGED
@@ -37,7 +37,7 @@ Simply use RubyGems:
37
37
 
38
38
  [s3_bucket (required)] S3 bucket name.
39
39
 
40
- [s3_endpoint] s3 endpoint name. Example, Tokyo region is "s3-ap-northeast-1.amazonaws.com".
40
+ [s3_endpoint] s3 endpoint name. For example, US West (Oregon) Region is "s3-us-west-2.amazonaws.com". The full list of endpoints are available here. > http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
41
41
 
42
42
  [s3_object_key_format] The format of S3 object keys. You can use several built-in variables:
43
43
 
@@ -79,8 +79,19 @@ The {fluent-mixin-config-placeholders}[https://github.com/tagomoris/fluent-mixin
79
79
 
80
80
  s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
81
81
 
82
+ [store_as] archive format on S3. You can use serveral format:
83
+
84
+ - gzip (default)
85
+ - json
86
+ - text
87
+ - lzo (Need lzop command)
88
+
82
89
  [auto_create_bucket] Create S3 bucket if it does not exists. Default is true.
83
90
 
91
+ [check_apikey_on_start] Check AWS key on start. Default is true.
92
+
93
+ [proxy_uri] uri of proxy environment.
94
+
84
95
  [path] path prefix of the files on S3. Default is "" (no prefix).
85
96
 
86
97
  [buffer_path (required)] path prefix of the files to buffer logs.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.3.1
1
+ 0.3.3
@@ -11,6 +11,7 @@ class S3Output < Fluent::TimeSlicedOutput
11
11
  require 'zlib'
12
12
  require 'time'
13
13
  require 'tempfile'
14
+ require 'open3'
14
15
 
15
16
  @use_ssl = true
16
17
  end
@@ -31,6 +32,8 @@ class S3Output < Fluent::TimeSlicedOutput
31
32
  config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
32
33
  config_param :store_as, :string, :default => "gzip"
33
34
  config_param :auto_create_bucket, :bool, :default => true
35
+ config_param :check_apikey_on_start, :bool, :default => true
36
+ config_param :proxy_uri, :string, :default => nil
34
37
 
35
38
  attr_reader :bucket
36
39
 
@@ -62,6 +65,13 @@ class S3Output < Fluent::TimeSlicedOutput
62
65
 
63
66
  @ext, @mime_type = case @store_as
64
67
  when 'gzip' then ['gz', 'application/x-gzip']
68
+ when 'lzo' then
69
+ begin
70
+ Open3.capture3('lzop -V')
71
+ rescue Errno::ENOENT
72
+ raise ConfigError, "'lzop' utility must be in PATH for LZO compression"
73
+ end
74
+ ['lzo', 'application/x-lzop']
65
75
  when 'json' then ['json', 'application/json']
66
76
  else ['txt', 'text/plain']
67
77
  end
@@ -77,13 +87,14 @@ class S3Output < Fluent::TimeSlicedOutput
77
87
  options[:secret_access_key] = @aws_sec_key
78
88
  end
79
89
  options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
90
+ options[:proxy_uri] = @proxy_uri if @proxy_uri
80
91
  options[:use_ssl] = @use_ssl
81
92
 
82
93
  @s3 = AWS::S3.new(options)
83
94
  @bucket = @s3.buckets[@s3_bucket]
84
95
 
85
96
  ensure_bucket
86
- check_apikeys
97
+ check_apikeys if @check_apikey_on_start
87
98
  end
88
99
 
89
100
  def format(tag, time, record)
@@ -128,6 +139,13 @@ class S3Output < Fluent::TimeSlicedOutput
128
139
  w = Zlib::GzipWriter.new(tmp)
129
140
  chunk.write_to(w)
130
141
  w.close
142
+ elsif @store_as == "lzo"
143
+ w = Tempfile.new("chunk-tmp")
144
+ chunk.write_to(w)
145
+ w.close
146
+ tmp.close
147
+ # We don't check the return code because we can't recover lzop failure.
148
+ system "lzop -qf1 -o #{tmp.path} #{w.path}"
131
149
  else
132
150
  chunk.write_to(tmp)
133
151
  tmp.close
@@ -136,6 +154,7 @@ class S3Output < Fluent::TimeSlicedOutput
136
154
  ensure
137
155
  tmp.close(true) rescue nil
138
156
  w.close rescue nil
157
+ w.unlink rescue nil
139
158
  end
140
159
  end
141
160
 
data/test/out_s3.rb CHANGED
@@ -59,6 +59,17 @@ class S3OutputTest < Test::Unit::TestCase
59
59
  assert_equal 'text/plain', d.instance.instance_variable_get(:@mime_type)
60
60
  end
61
61
 
62
+ def test_configure_with_mime_type_lzo
63
+ conf = CONFIG.clone
64
+ conf << "\nstore_as lzo\n"
65
+ d = create_driver(conf)
66
+ assert_equal 'lzo', d.instance.instance_variable_get(:@ext)
67
+ assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@mime_type)
68
+ rescue => e
69
+ # TODO: replace code with disable lzop command
70
+ assert(e.is_a?(Fluent::ConfigError))
71
+ end
72
+
62
73
  def test_format
63
74
  d = create_driver
64
75
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.3.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-03-28 00:00:00.000000000 Z
12
+ date: 2013-06-18 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -136,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
136
136
  version: '0'
137
137
  segments:
138
138
  - 0
139
- hash: -4273077554576087642
139
+ hash: -3746525337841491520
140
140
  required_rubygems_version: !ruby/object:Gem::Requirement
141
141
  none: false
142
142
  requirements:
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
145
145
  version: '0'
146
146
  segments:
147
147
  - 0
148
- hash: -4273077554576087642
148
+ hash: -3746525337841491520
149
149
  requirements: []
150
150
  rubyforge_project:
151
151
  rubygems_version: 1.8.23