fluent-plugin-s3 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cce40365e053256475e60bf82f122c1ed0ccff27
-  data.tar.gz: e39318c0a195293034c053a457f455ab93feda5d
+  metadata.gz: 706ac1da3860e8310ea63b3a0acdb3d819f327ed
+  data.tar.gz: a5dc2d7ed59f8af1f1ecbba02cbb6b8b49b9f256
 SHA512:
-  metadata.gz: e112e247a473d37fbaaebef78f43a0aa1c81b1b29d667be08a8bf81e876ca5effb8b0cc6d8768019a3163eb463f059d6dbab8cf3fc414cece916666602893b73
-  data.tar.gz: 609f60be06d0ecfc986cef522078da31d78494d9a1ebb036b0442961159f31dc4ba711008547bd374c41924312d68ffb303f258e0c160b0c382a968497ae8b11
+  metadata.gz: de3ef963574a32ea2ccb517e7ea6e6a8e17b8489e2b8809f8ba3a243d695f4d0e14a5e91d238b93fa703c892a487bf0f9a172870ac249cef438b98e5edd707e9
+  data.tar.gz: c2e79fb312a87fddda7c5d8dfef25448c59d3923a89f03c904e93891aa203a13c3f174ba087d3da8edc69b5146cc0ee137b6fd9a1a7b727e68dbe0710a24cde0

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 1.1.8 - 2019/01/28
+
+  * in_s3: Restart SQS polling when the error happens
+  * out_s3: Show root cause when error happens during compressor loading
+
 Release 1.1.7 - 2018/11/14
 
   * in_s3: Fix the bug proxy_uri parameter setting

data/README.md

@@ -282,11 +282,9 @@ to decide keys dynamically.
 * %{path} is exactly the value of **path** configured in the configuration file.
 E.g., "logs/" in the example configuration above.
 * %{time_slice} is the
-time-slice in text that are formatted with **time_slice_format**. %{index} is
-the sequential number starts from 0, increments when multiple files are
-uploaded to S3 in the same time slice.
-* %{file_extention} is always "gz" for
-now.
+time-slice in text that are formatted with **time_slice_format**.
+* %{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
+* %{file_extension} depends on **store_as** parameter.
 * %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed. If you want to use this placeholder, install `uuidtools` gem first.
 * %{hostname} is replaced with `Socket.gethostname` result.
 * %{hex_random} a random hex string that is replaced for each buffer chunk, not
@@ -652,6 +650,10 @@ When true, messages are not deleted after polling block. Default is false.
 
 The long polling interval. Default is 20.
 
+**sqs/retry_error_interval**
+
+Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 300.
+
 ## IAM Policy
 
 The following is an example for a minimal IAM policy needed to write to an s3

data/VERSION

@@ -1 +1 @@
-1.1.7
+1.1.8

data/lib/fluent/plugin/in_s3.rb

@@ -86,6 +86,8 @@ module Fluent::Plugin
       config_param :skip_delete, :bool, default: false
       desc "The long polling interval."
       config_param :wait_time_seconds, :integer, default: 20
+      desc "Polling error retry interval."
+      config_param :retry_error_interval, :integer, default: 300
     end
 
     desc "Tag string"
@@ -159,18 +161,24 @@ module Fluent::Plugin
       @poller.before_request do |stats|
         throw :stop_polling unless @running
       end
-      @poller.poll(options) do |message|
-        begin
-          body = Yajl.load(message.body)
-          log.debug(body)
-          next unless body["Records"] # skip test queue
-
-          process(body)
-        rescue => e
-          log.warn(error: e)
-          log.warn_backtrace(e.backtrace)
-          throw :skip_delete
+      begin
+        @poller.poll(options) do |message|
+          begin
+            body = Yajl.load(message.body)
+            log.debug(body)
+            next unless body["Records"] # skip test queue
+
+            process(body)
+          rescue => e
+            log.warn(error: e)
+            log.warn_backtrace(e.backtrace)
+            throw :skip_delete
+          end
         end
+      rescue => e
+        log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds")
+        sleep(@sqs.retry_error_interval)
+        retry
       end
     end
 

data/lib/fluent/plugin/out_s3.rb

@@ -152,8 +152,8 @@ module Fluent::Plugin
       begin
         buffer_type = @buffer_config[:@type]
         @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, log: log)
-      rescue
-        log.warn "#{@store_as} not found. Use 'text' instead"
+      rescue => e
+        log.warn "'#{@store_as}' not supported. Use 'text' instead: error = #{e.message}"
         @compressor = TextCompressor.new
       end
       @compressor.configure(conf)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-15 00:00:00.000000000 Z
+date: 2019-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd