fluent-plugin-s3 0.8.0.rc1 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 396e2f615eeaf603b9240f364dea07db8a7e92ef
4
- data.tar.gz: 64b0a5e72ac0ee484e676d15417c8c44536001c9
3
+ metadata.gz: fa201801e95642044876a29fe308d0df2912c214
4
+ data.tar.gz: a04b2703154c2d7efd698911bb73a04e22ca0d8c
5
5
  SHA512:
6
- metadata.gz: 3bc5c5e87a48959943863e011ea8d12532b348602cb94f76c6c050af4e07bafd7d6f175e7a1c5f111187a8c7d63e994a98983f50a3af168550c7dc3f6644e137
7
- data.tar.gz: 4763fb32496ccb0cea8565aea1b2b553fefa015cfaf952f31f7eb7d15a5361ba46a2545c8fbe4296c2d63e6de1290263c07d9cce265935de5bca5236e0939136
6
+ metadata.gz: 425e1f44dcf826ee015b0b6581ed580648419dab9562a7ea5b2afe5dbb25cd278767e4c0be2ccab3aeb9eaa233a86a92c7721a11c6a1c6487e0f8b302736c11e
7
+ data.tar.gz: 23c621c0922ad06e43766bc58c92e203f389c09b6b3a1d03c7ee4c3fa186368ae6032c9ed5cfa568bcde0f3ed2435257cc0353d7f0a5530abccc980132e3e861
data/.travis.yml CHANGED
@@ -6,6 +6,7 @@ rvm:
6
6
  - 2.1
7
7
  - 2.2.3
8
8
  - 2.3.1
9
+ - 2.4.0-rc1
9
10
  - ruby-head
10
11
  - rbx
11
12
 
@@ -32,3 +33,5 @@ matrix:
32
33
  gemfile: Gemfile
33
34
  - rvm: 2.0
34
35
  gemfile: Gemfile
36
+ - rvm: 2.4.0-rc1
37
+ gemfile: Gemfile.v0.12
data/ChangeLog CHANGED
@@ -1,3 +1,9 @@
1
+ Release 0.8.0 - 2016/12/20
2
+
3
+ * out_s3: Add check_object / check_bucket parameters for only put permission
4
+ * Remove fluent-mixin-config-placeholders dependency
5
+
6
+
1
7
  Release 0.7.2 - 2016/10/20
2
8
 
3
9
  * in_s3: Replace old parser API with new one
data/Gemfile CHANGED
@@ -1,7 +1,5 @@
1
1
  source "http://rubygems.org"
2
2
 
3
- gem 'json', '= 1.8.3'
4
3
  gem 'uuidtools'
5
- gem 'fluentd', '= 0.14.8'
6
4
 
7
5
  gemspec
data/README.md CHANGED
@@ -209,6 +209,34 @@ You can change key name by "message_key" option.
209
209
 
210
210
  Create S3 bucket if it does not exists. Default is true.
211
211
 
212
+ **check_bukcet**
213
+
214
+ Check mentioned bucket if it exists in AWS or not. Default is true.
215
+
216
+ When it is false,
217
+ fluentd will not check aws s3 for the existence of the mentioned bucket. This is the
218
+ case where bucket will be pre-created before running fluentd.
219
+
220
+ **check_object**
221
+
222
+ Check object before creation if it exists or not. Default is true.
223
+
224
+ When it is false,
225
+ s3_object_key_format will be %{path}%{date_slice}_%{time_slice}.%{file_extension}
226
+ where, time_slice will be in hhmmss format, so that each object will be unique.
227
+ Example object name, assuming it is created on 2016/16/11 3:30:54 PM
228
+ 20161611_153054.txt (extension can be anything as per user's choice)
229
+
230
+ **Example when check_bukcet=false and check_object=false**
231
+
232
+ When the mentioned configuration will be made, fluentd will work with the
233
+ minimum IAM poilcy, like:
234
+ "Statement": [{
235
+ "Effect": "Allow",
236
+ "Action": "s3:PutObject",
237
+ "Resource": ["*"]
238
+ }]
239
+
212
240
  **check_apikey_on_start**
213
241
 
214
242
  Check AWS key on start. Default is true.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.8.0.rc1
1
+ 0.8.0
@@ -101,6 +101,10 @@ module Fluent
101
101
  config_param :hex_random_length, :integer, :default => 4
102
102
  desc "Overwrite already existing path"
103
103
  config_param :overwrite, :bool, :default => false
104
+ desc "Check bucket if exists or not"
105
+ config_param :check_bucket, :bool, :default => true
106
+ desc "Check object before creation"
107
+ config_param :check_object, :bool, :default => true
104
108
  desc "Specifies the AWS KMS key ID to use for object encryption"
105
109
  config_param :ssekms_key_id, :string, :default => nil, :secret => true
106
110
  desc "Specifies the algorithm to use to when encrypting the object"
@@ -175,7 +179,11 @@ module Fluent
175
179
  @bucket = @s3.bucket(@s3_bucket)
176
180
 
177
181
  check_apikeys if @check_apikey_on_start
178
- ensure_bucket
182
+ ensure_bucket if @check_bucket
183
+
184
+ if !@check_object
185
+ @s3_object_key_format = "%{path}/%{date_slice}_%{hms_slice}.%{file_extension}"
186
+ end
179
187
 
180
188
  super
181
189
  end
@@ -188,35 +196,53 @@ module Fluent
188
196
  i = 0
189
197
  previous_path = nil
190
198
 
191
- begin
192
- path = @path_slicer.call(@path)
199
+ if @check_object
200
+ begin
201
+ path = @path_slicer.call(@path)
202
+
203
+ @values_for_s3_object_chunk[chunk.unique_id] ||= {
204
+ "hex_random" => hex_random(chunk),
205
+ }
206
+ values_for_s3_object_key = {
207
+ "path" => path,
208
+ "time_slice" => chunk.key,
209
+ "file_extension" => @compressor.ext,
210
+ "index" => i,
211
+ }.merge!(@values_for_s3_object_chunk[chunk.unique_id])
212
+ values_for_s3_object_key['uuid_flush'.freeze] = uuid_random if @uuid_flush_enabled
213
+
214
+ s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
215
+ values_for_s3_object_key[expr[2...expr.size-1]]
216
+ }
217
+ if (i > 0) && (s3path == previous_path)
218
+ if @overwrite
219
+ log.warn "#{s3path} already exists, but will overwrite"
220
+ break
221
+ else
222
+ raise "duplicated path is generated. use %{index} in s3_object_key_format: path = #{s3path}"
223
+ end
224
+ end
225
+
226
+ i += 1
227
+ previous_path = s3path
228
+ end while @bucket.object(s3path).exists?
229
+ else
230
+ if @localtime
231
+ hms_slicer = Time.now.strftime("%H%M%S")
232
+ else
233
+ hms_slicer = Time.now.utc.strftime("%H%M%S")
234
+ end
193
235
 
194
- @values_for_s3_object_chunk[chunk.unique_id] ||= {
195
- "hex_random" => hex_random(chunk),
196
- }
197
236
  values_for_s3_object_key = {
198
- "path" => path,
199
- "time_slice" => chunk.key,
237
+ "path" => @path_slicer.call(@path),
238
+ "date_slice" => chunk.key,
200
239
  "file_extension" => @compressor.ext,
201
- "index" => i,
202
- }.merge!(@values_for_s3_object_chunk[chunk.unique_id])
203
- values_for_s3_object_key['uuid_flush'.freeze] = uuid_random if @uuid_flush_enabled
204
-
240
+ "hms_slice" => hms_slicer,
241
+ }
205
242
  s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
206
243
  values_for_s3_object_key[expr[2...expr.size-1]]
207
244
  }
208
- if (i > 0) && (s3path == previous_path)
209
- if @overwrite
210
- log.warn "#{s3path} already exists, but will overwrite"
211
- break
212
- else
213
- raise "duplicated path is generated. use %{index} in s3_object_key_format: path = #{s3path}"
214
- end
215
- end
216
-
217
- i += 1
218
- previous_path = s3path
219
- end while @bucket.object(s3path).exists?
245
+ end
220
246
 
221
247
  tmp = Tempfile.new("s3-")
222
248
  tmp.binmode
data/test/test_in_s3.rb CHANGED
@@ -21,7 +21,6 @@ class S3InputTest < Test::Unit::TestCase
21
21
  aws_key_id test_key_id
22
22
  aws_sec_key test_sec_key
23
23
  s3_bucket test_bucket
24
- utc
25
24
  buffer_type memory
26
25
  <sqs>
27
26
  queue_name test_queue
@@ -70,7 +69,6 @@ class S3InputTest < Test::Unit::TestCase
70
69
  aws_key_id test_key_id
71
70
  aws_sec_key test_sec_key
72
71
  s3_bucket test_bucket
73
- utc
74
72
  ]
75
73
  assert_raise_message("'<sqs>' sections are required") do
76
74
  create_driver(conf)
data/test/test_out_s3.rb CHANGED
@@ -52,6 +52,8 @@ class S3OutputTest < Test::Unit::TestCase
52
52
  assert_equal false, d.instance.force_path_style
53
53
  assert_equal nil, d.instance.compute_checksums
54
54
  assert_equal nil, d.instance.signature_version
55
+ assert_equal true, d.instance.check_bucket
56
+ assert_equal true, d.instance.check_object
55
57
  end
56
58
 
57
59
  def test_s3_endpoint_with_valid_endpoint
@@ -119,6 +121,14 @@ class S3OutputTest < Test::Unit::TestCase
119
121
  end
120
122
  end
121
123
 
124
+ def test_configure_with_no_check_on_s3
125
+ conf = CONFIG.clone
126
+ conf << "\ncheck_bucket false\ncheck_object false\n"
127
+ d = create_driver(conf)
128
+ assert_equal false, d.instance.check_bucket
129
+ assert_equal false, d.instance.check_object
130
+ end
131
+
122
132
  def test_path_slicing
123
133
  config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
124
134
  d = create_driver(config)
@@ -260,6 +270,8 @@ class S3OutputTest < Test::Unit::TestCase
260
270
  utc
261
271
  buffer_type memory
262
272
  log_level debug
273
+ check_bucket true
274
+ check_object true
263
275
  ]
264
276
 
265
277
  def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
@@ -272,6 +284,37 @@ class S3OutputTest < Test::Unit::TestCase
272
284
  d
273
285
  end
274
286
 
287
+ def test_write_with_hardened_s3_policy
288
+ # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
289
+ setup_mocks_hardened_policy
290
+ s3_local_file_path = "/tmp/s3-test.txt"
291
+ # @s3_object_key_format will be hard_coded with timestamp only,
292
+ # as in this case, it will not check for object existence, not even bucker existence
293
+ # check_bukcet and check_object both of this config parameter should be false
294
+ # @s3_object_key_format = "%{path}/%{time_slice}_%{hms_slice}.%{file_extension}"
295
+ setup_s3_object_mocks_hardened_policy()
296
+
297
+ # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
298
+ # to make assertions on chunks' keys
299
+ config = CONFIG_TIME_SLICE.gsub(/check_object true/, "check_object false\n")
300
+ config = config.gsub(/check_bucket true/, "check_bucket false\n")
301
+ d = create_time_sliced_driver(config)
302
+
303
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
304
+ d.emit({"a"=>1}, time)
305
+ d.emit({"a"=>2}, time)
306
+
307
+ # Finally, the instance of S3Output is initialized and then invoked
308
+ d.run
309
+ Zlib::GzipReader.open(s3_local_file_path) do |gz|
310
+ data = gz.read
311
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
312
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
313
+ data
314
+ end
315
+ FileUtils.rm_f(s3_local_file_path)
316
+ end
317
+
275
318
  def test_write_with_custom_s3_object_key_format
276
319
  # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
277
320
  setup_mocks(true)
@@ -404,6 +447,36 @@ class S3OutputTest < Test::Unit::TestCase
404
447
  @s3_bucket.object(s3path) { s3obj }
405
448
  end
406
449
 
450
+ def setup_mocks_hardened_policy()
451
+ @s3_client = stub(Aws::S3::Client.new(:stub_responses => true))
452
+ mock(Aws::S3::Client).new(anything).at_least(0) { @s3_client }
453
+ @s3_resource = mock(Aws::S3::Resource.new(:client => @s3_client))
454
+ mock(Aws::S3::Resource).new(:client => @s3_client) { @s3_resource }
455
+ @s3_bucket = mock(Aws::S3::Bucket.new(:name => "test",
456
+ :client => @s3_client))
457
+ @s3_object = mock(Aws::S3::Object.new(:bucket_name => "test_bucket",
458
+ :key => "test",
459
+ :client => @s3_client))
460
+ @s3_bucket.object(anything).at_least(0) { @s3_object }
461
+ @s3_resource.bucket(anything) { @s3_bucket }
462
+ end
463
+
464
+ def setup_s3_object_mocks_hardened_policy(params = {})
465
+ s3path = params[:s3path] || "log/20110102_131415.gz"
466
+ s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
467
+
468
+ # Assert content of event logs which are being sent to S3
469
+ s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
470
+ :key => "test",
471
+ :client => @s3_client))
472
+
473
+ tempfile = File.new(s3_local_file_path, "w")
474
+ stub(Tempfile).new("s3-") { tempfile }
475
+ s3obj.put(:body => tempfile,
476
+ :content_type => "application/x-gzip",
477
+ :storage_class => "STANDARD")
478
+ end
479
+
407
480
  def test_auto_create_bucket_false_with_non_existence_bucket
408
481
  setup_mocks
409
482
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0.rc1
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-11-24 00:00:00.000000000 Z
12
+ date: 2016-12-21 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -163,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
163
163
  version: '0'
164
164
  required_rubygems_version: !ruby/object:Gem::Requirement
165
165
  requirements:
166
- - - ">"
166
+ - - ">="
167
167
  - !ruby/object:Gem::Version
168
- version: 1.3.1
168
+ version: '0'
169
169
  requirements: []
170
170
  rubyforge_project:
171
171
  rubygems_version: 2.5.1