fluent-plugin-s3 1.1.11 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e034c8d5e4ae06a88f405bfe396d2c1c960da096b5010d4b45902765b00eaa6a
-  data.tar.gz: 679865ce71becff176745c0fcb051a6120313c5127d4ad70a8333b263c5eb67d
+  metadata.gz: 26e0fd779938910c499dfdd043c253b90167c05c585945fddcaa37985302cbff
+  data.tar.gz: fe09d0a34580eeea82b3538f7e7fe8fcbc85e42dcb1d7dd5cedfbfeeefdfbe56
 SHA512:
-  metadata.gz: b0e0bf4a2072b9589e0e27ed84806c00d1fcc1ba47e70d1a628017aa7981993372b2afd7ee04d1eee5c558c2d485c3128ec64318b34fd802b8c1759fa311d71d
-  data.tar.gz: 6fd057fd51ce0aeed1fd64c2a39a405ee329f0b96cb92dc09faeac37f4d679fc0d7951d827f8ea0b0eda513a99f9cd89ab171b795933b1d8a44294d025fac7c8
+  metadata.gz: aec5991826999db113542e9e313b03a2ca018d3b7143ca745f45a211f0ce3e3c66c1b5b84c087b3e5d654aeb8b840db1580e5b6911bd182b7fb1a4131ee317ce
+  data.tar.gz: 575fcd019697834d1ff5aef35a456fa7b5a54ec8c3f565285b50195ca1f5d67a24738d39973e9f7d7e9e15bac398ac96d494ffc1907e5cd6271385242cf36521

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.2.0 - 2019/10/17
+
+  * out_s3: Add bucket_lifecycle_rule section to set bucket's lifecycle
+
 Release 1.1.11 - 2019/06/17
 
   * in_s3: Add add_object_metadata parameter

data/README.md

@@ -5,9 +5,6 @@ alt="Build Status" />](https://travis-ci.org/fluent/fluent-plugin-s3) [<img
 src="https://codeclimate.com/github/fluent/fluent-plugin-s3/badges/gpa.svg"
 />](https://codeclimate.com/github/fluent/fluent-plugin-s3)
 
-## PR
-**it will be greatful that the plugin can push log to  owner S3 Storage like Ceph S3.**
-
 ## Overview
 
 **s3** output plugin buffers event logs in local file and upload it to S3
@@ -569,6 +566,16 @@ It would be useful when you use S3 compatible storage that accepts only signatur
 
 Given a threshold to treat events as delay, output warning logs if delayed events were put into s3.
 
+**bucket_lifecycle_rule**
+
+Specify one or more lifecycle rules for the bucket
+
+    <bucket_lifecycle_rule>
+      id UNIQUE_ID_FOR_THE_RULE
+      prefix OPTIONAL_PREFIX # Objects whose keys begin with this prefix will be affected by the rule. If not specified all objects of the bucket will be affected
+      expiration_days NUMBER_OF_DAYS # The number of days before the object will expire
+    </bucket_lifecycle_rule>
+
 ## Input: Setup
 
 1. Create new [SQS](https://aws.amazon.com/documentation/sqs/) queue (use same region as S3)

data/VERSION

@@ -1 +1 @@
-1.1.11
+1.2.0

data/lib/fluent/plugin/out_s3.rb

@@ -124,6 +124,14 @@ module Fluent::Plugin
     config_param :warn_for_delay, :time, default: nil
     desc "Arbitrary S3 metadata headers to set for the object"
     config_param :s3_metadata, :hash, default: nil
+    config_section :bucket_lifecycle_rule, param_name: :bucket_lifecycle_rules, multi: true do
+      desc "A unique ID for this rule"
+      config_param :id, :string
+      desc "Objects whose keys begin with this prefix will be affected by the rule. If not specified all objects of the bucket will be affected"
+      config_param :prefix, :string, default: ''
+      desc "The number of days before the object will expire"
+      config_param :expiration_days, :integer
+    end
 
     DEFAULT_FORMAT_TYPE = "out_file"
 
@@ -217,6 +225,7 @@ module Fluent::Plugin
 
       check_apikeys if @check_apikey_on_start
       ensure_bucket if @check_bucket
+      ensure_bucket_lifecycle
 
       super
     end
@@ -373,6 +382,30 @@ module Fluent::Plugin
       end
     end
 
+    def ensure_bucket_lifecycle
+      unless @bucket_lifecycle_rules.empty?
+        old_rules = get_bucket_lifecycle_rules
+        new_rules = @bucket_lifecycle_rules.sort_by { |rule| rule.id }.map do |rule|
+          { id: rule.id, expiration: { days: rule.expiration_days }, prefix: rule.prefix, status: "Enabled" }
+        end
+
+        unless old_rules == new_rules
+          log.info "Configuring bucket lifecycle rules for #{@s3_bucket} on #{@s3_endpoint}"
+          @bucket.lifecycle_configuration.put({ lifecycle_configuration: { rules: new_rules } })
+        end
+      end
+    end
+
+    def get_bucket_lifecycle_rules
+      begin
+        @bucket.lifecycle_configuration.rules.sort_by { |rule| rule[:id] }.map do |rule|
+          { id: rule[:id], expiration: { days: rule[:expiration][:days] }, prefix: rule[:prefix], status: rule[:status] }
+        end
+      rescue Aws::S3::Errors::NoSuchLifecycleConfiguration
+        []
+      end
+    end
+
     def process_s3_object_key_format
       %W(%{uuid} %{uuid:random} %{uuid:hostname} %{uuid:timestamp}).each { |ph|
         if @s3_object_key_format.include?(ph)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.1.11
+  version: 1.2.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-17 00:00:00.000000000 Z
+date: 2019-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd