fluent-plugin-s3 0.8.0.rc1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 396e2f615eeaf603b9240f364dea07db8a7e92ef
-  data.tar.gz: 64b0a5e72ac0ee484e676d15417c8c44536001c9
+  metadata.gz: fa201801e95642044876a29fe308d0df2912c214
+  data.tar.gz: a04b2703154c2d7efd698911bb73a04e22ca0d8c
 SHA512:
-  metadata.gz: 3bc5c5e87a48959943863e011ea8d12532b348602cb94f76c6c050af4e07bafd7d6f175e7a1c5f111187a8c7d63e994a98983f50a3af168550c7dc3f6644e137
-  data.tar.gz: 4763fb32496ccb0cea8565aea1b2b553fefa015cfaf952f31f7eb7d15a5361ba46a2545c8fbe4296c2d63e6de1290263c07d9cce265935de5bca5236e0939136
+  metadata.gz: 425e1f44dcf826ee015b0b6581ed580648419dab9562a7ea5b2afe5dbb25cd278767e4c0be2ccab3aeb9eaa233a86a92c7721a11c6a1c6487e0f8b302736c11e
+  data.tar.gz: 23c621c0922ad06e43766bc58c92e203f389c09b6b3a1d03c7ee4c3fa186368ae6032c9ed5cfa568bcde0f3ed2435257cc0353d7f0a5530abccc980132e3e861

data/.travis.yml

@@ -6,6 +6,7 @@ rvm:
   - 2.1
   - 2.2.3
   - 2.3.1
+  - 2.4.0-rc1
   - ruby-head
   - rbx
 
@@ -32,3 +33,5 @@ matrix:
       gemfile: Gemfile
     - rvm: 2.0
       gemfile: Gemfile
+    - rvm: 2.4.0-rc1
+      gemfile: Gemfile.v0.12

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.8.0 - 2016/12/20
+
+  * out_s3: Add check_object / check_bucket parameters for only put permission
+  * Remove fluent-mixin-config-placeholders dependency
+
+
 Release 0.7.2 - 2016/10/20
 
   * in_s3: Replace old parser API with new one

data/Gemfile

@@ -1,7 +1,5 @@
 source "http://rubygems.org"
 
-gem 'json', '= 1.8.3'
 gem 'uuidtools'
-gem 'fluentd', '= 0.14.8'
 
 gemspec

data/README.md

@@ -209,6 +209,34 @@ You can change key name by "message_key" option.
 
 Create S3 bucket if it does not exists. Default is true.
 
+**check_bukcet**
+
+Check mentioned bucket if it exists in AWS or not. Default is true.
+
+When it is false,
+	fluentd will not check aws s3 for the existence of the mentioned bucket. This is the
+	case where bucket will be pre-created before running fluentd.
+
+**check_object**
+
+Check object before creation if it exists or not. Default is true.
+
+When it is false,
+	s3_object_key_format will be %{path}%{date_slice}_%{time_slice}.%{file_extension}
+	where, time_slice will be in hhmmss format, so that each object will be unique.
+	Example object name, assuming it is created on 2016/16/11 3:30:54 PM
+		20161611_153054.txt (extension can be anything as per user's choice)
+
+**Example when check_bukcet=false and check_object=false**
+
+When the mentioned configuration will be made, fluentd will work with the
+minimum IAM poilcy, like:
+				"Statement": [{
+					"Effect": "Allow",
+					"Action": "s3:PutObject",
+					"Resource": ["*"]
+				}]
+
 **check_apikey_on_start**
 
 Check AWS key on start. Default is true.

data/VERSION

@@ -1 +1 @@
-0.8.0.rc1
+0.8.0

data/lib/fluent/plugin/out_s3.rb

@@ -101,6 +101,10 @@ module Fluent
     config_param :hex_random_length, :integer, :default => 4
     desc "Overwrite already existing path"
     config_param :overwrite, :bool, :default => false
+    desc "Check bucket if exists or not"
+    config_param :check_bucket, :bool, :default => true
+    desc "Check object before creation"
+    config_param :check_object, :bool, :default => true
     desc "Specifies the AWS KMS key ID to use for object encryption"
     config_param :ssekms_key_id, :string, :default => nil, :secret => true
     desc "Specifies the algorithm to use to when encrypting the object"
@@ -175,7 +179,11 @@ module Fluent
       @bucket = @s3.bucket(@s3_bucket)
 
       check_apikeys if @check_apikey_on_start
-      ensure_bucket
+      ensure_bucket if @check_bucket
+
+      if !@check_object
+        @s3_object_key_format = "%{path}/%{date_slice}_%{hms_slice}.%{file_extension}"
+      end
 
       super
     end
@@ -188,35 +196,53 @@ module Fluent
       i = 0
       previous_path = nil
 
-      begin
-        path = @path_slicer.call(@path)
+      if @check_object
+        begin
+          path = @path_slicer.call(@path)
+
+          @values_for_s3_object_chunk[chunk.unique_id] ||= {
+            "hex_random" => hex_random(chunk),
+          }
+          values_for_s3_object_key = {
+            "path" => path,
+            "time_slice" => chunk.key,
+            "file_extension" => @compressor.ext,
+            "index" => i,
+          }.merge!(@values_for_s3_object_chunk[chunk.unique_id])
+          values_for_s3_object_key['uuid_flush'.freeze] = uuid_random if @uuid_flush_enabled
+
+          s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
+            values_for_s3_object_key[expr[2...expr.size-1]]
+          }
+          if (i > 0) && (s3path == previous_path)
+            if @overwrite
+              log.warn "#{s3path} already exists, but will overwrite"
+              break
+            else
+              raise "duplicated path is generated. use %{index} in s3_object_key_format: path = #{s3path}"
+            end
+          end
+
+          i += 1
+          previous_path = s3path
+        end while @bucket.object(s3path).exists?
+      else
+        if @localtime
+          hms_slicer = Time.now.strftime("%H%M%S")
+        else
+          hms_slicer = Time.now.utc.strftime("%H%M%S")
+        end
 
-        @values_for_s3_object_chunk[chunk.unique_id] ||= {
-          "hex_random" => hex_random(chunk),
-        }
         values_for_s3_object_key = {
-          "path" => path,
-          "time_slice" => chunk.key,
+          "path" => @path_slicer.call(@path),
+          "date_slice" => chunk.key,
           "file_extension" => @compressor.ext,
-          "index" => i,
-        }.merge!(@values_for_s3_object_chunk[chunk.unique_id])
-        values_for_s3_object_key['uuid_flush'.freeze] = uuid_random if @uuid_flush_enabled
-
+          "hms_slice" => hms_slicer,
+        }
         s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
           values_for_s3_object_key[expr[2...expr.size-1]]
         }
-        if (i > 0) && (s3path == previous_path)
-          if @overwrite
-            log.warn "#{s3path} already exists, but will overwrite"
-            break
-          else
-            raise "duplicated path is generated. use %{index} in s3_object_key_format: path = #{s3path}"
-          end
-        end
-
-        i += 1
-        previous_path = s3path
-      end while @bucket.object(s3path).exists?
+      end
 
       tmp = Tempfile.new("s3-")
       tmp.binmode

data/test/test_in_s3.rb

@@ -21,7 +21,6 @@ class S3InputTest < Test::Unit::TestCase
     aws_key_id test_key_id
     aws_sec_key test_sec_key
     s3_bucket test_bucket
-    utc
     buffer_type memory
     <sqs>
       queue_name test_queue
@@ -70,7 +69,6 @@ class S3InputTest < Test::Unit::TestCase
         aws_key_id test_key_id
         aws_sec_key test_sec_key
         s3_bucket test_bucket
-        utc
       ]
       assert_raise_message("'<sqs>' sections are required") do
         create_driver(conf)

data/test/test_out_s3.rb

@@ -52,6 +52,8 @@ class S3OutputTest < Test::Unit::TestCase
     assert_equal false, d.instance.force_path_style
     assert_equal nil, d.instance.compute_checksums
     assert_equal nil, d.instance.signature_version
+    assert_equal true, d.instance.check_bucket
+    assert_equal true, d.instance.check_object
   end
 
   def test_s3_endpoint_with_valid_endpoint
@@ -119,6 +121,14 @@ class S3OutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_configure_with_no_check_on_s3
+    conf = CONFIG.clone
+    conf << "\ncheck_bucket false\ncheck_object false\n"
+    d = create_driver(conf)
+    assert_equal false, d.instance.check_bucket
+    assert_equal false, d.instance.check_object
+  end
+
   def test_path_slicing
     config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
     d = create_driver(config)
@@ -260,6 +270,8 @@ class S3OutputTest < Test::Unit::TestCase
     utc
     buffer_type memory
     log_level debug
+    check_bucket true
+    check_object true
   ]
 
   def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
@@ -272,6 +284,37 @@ class S3OutputTest < Test::Unit::TestCase
     d
   end
 
+  def test_write_with_hardened_s3_policy
+    # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
+    setup_mocks_hardened_policy
+    s3_local_file_path = "/tmp/s3-test.txt"
+    # @s3_object_key_format will be hard_coded with timestamp only,
+    # as in this case, it will not check for object existence, not even bucker existence
+    # check_bukcet and check_object both of this config parameter should be false
+    # @s3_object_key_format = "%{path}/%{time_slice}_%{hms_slice}.%{file_extension}"
+    setup_s3_object_mocks_hardened_policy()
+
+    # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
+    # to make assertions on chunks' keys
+    config = CONFIG_TIME_SLICE.gsub(/check_object true/, "check_object false\n")
+    config = config.gsub(/check_bucket true/, "check_bucket false\n")
+    d = create_time_sliced_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    # Finally, the instance of S3Output is initialized and then invoked
+    d.run
+    Zlib::GzipReader.open(s3_local_file_path) do |gz|
+      data = gz.read
+      assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                   %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                   data
+    end
+    FileUtils.rm_f(s3_local_file_path)
+  end
+
   def test_write_with_custom_s3_object_key_format
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
     setup_mocks(true)
@@ -404,6 +447,36 @@ class S3OutputTest < Test::Unit::TestCase
     @s3_bucket.object(s3path) { s3obj }
   end
 
+  def setup_mocks_hardened_policy()
+    @s3_client = stub(Aws::S3::Client.new(:stub_responses => true))
+    mock(Aws::S3::Client).new(anything).at_least(0) { @s3_client }
+    @s3_resource = mock(Aws::S3::Resource.new(:client => @s3_client))
+    mock(Aws::S3::Resource).new(:client => @s3_client) { @s3_resource }
+    @s3_bucket = mock(Aws::S3::Bucket.new(:name => "test",
+                                          :client => @s3_client))
+    @s3_object = mock(Aws::S3::Object.new(:bucket_name => "test_bucket",
+                                          :key => "test",
+                                          :client => @s3_client))
+    @s3_bucket.object(anything).at_least(0) { @s3_object }
+    @s3_resource.bucket(anything) { @s3_bucket }
+  end
+
+  def setup_s3_object_mocks_hardened_policy(params = {})
+    s3path = params[:s3path] || "log/20110102_131415.gz"
+    s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
+
+    # Assert content of event logs which are being sent to S3
+    s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
+                                     :key => "test",
+                                     :client => @s3_client))
+
+    tempfile = File.new(s3_local_file_path, "w")
+    stub(Tempfile).new("s3-") { tempfile }
+    s3obj.put(:body => tempfile,
+              :content_type => "application/x-gzip",
+              :storage_class => "STANDARD")
+  end
+
   def test_auto_create_bucket_false_with_non_existence_bucket
     setup_mocks
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.8.0.rc1
+  version: 0.8.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-24 00:00:00.000000000 Z
+date: 2016-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -163,9 +163,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1