fluent-plugin-s3 1.0.0.rc3 → 1.0.0.rc4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50b7e3d9177ef898ea2ee43c2ec789b2cb3293f2
-  data.tar.gz: 26ad4cbb951285caed1570ce28970344fbdf608a
+  metadata.gz: dbc4b32e04be1ba3836e5c34a09df8a831b42086
+  data.tar.gz: 0cf95c08b2eabc0f249f5a7e468cf6c597e0626b
 SHA512:
-  metadata.gz: 2bee78cb5c5bc10c157d232107bcfedc7225ee23b49987eb7b2eb1161f998731589d40ecaaabd5ca623462f367d83e3efdfe9ff21b6e95b2030ef7a9b0a4e4ce
-  data.tar.gz: 2f877c094484f090dc2d7a418553af0f3303528625d91e5e9c19350e0c4ea7a013f73926f301e5c8632f9e109fd0e36914ac1a973f714f554d17ff140cbed6a5
+  metadata.gz: f1c8dd2918e1c4490881a29a5e7d234f59e13ba0907a73a84dfd4a8b7e8a8f470b28a3cbaa97fefc116a4ca778c132c588d3503e4fab2eb7e5ffd65ea6b1aa94
+  data.tar.gz: ce4efb8bedd6f58d701677ae457383cae253ac2a46fadac01b5a3ef4a4bfeb1b6a001220babc01911a0beb42bc81ca3f141b11c86ae1a8c7f6cacd564b4c4df3

data/README.md

@@ -5,6 +5,9 @@ alt="Build Status" />](https://travis-ci.org/fluent/fluent-plugin-s3) [<img
 src="https://codeclimate.com/github/fluent/fluent-plugin-s3/badges/gpa.svg"
 />](https://codeclimate.com/github/fluent/fluent-plugin-s3)
 
+## PR
+**it will be greatful that the plugin can push log to  owner S3 Storage like Ceph S3.**
+
 ## Overview
 
 **s3** output plugin buffers event logs in local file and upload it to S3
@@ -74,7 +77,9 @@ For `<buffer>`, you can use any record field in `path` / `s3_object_key_format`.
       # parameters...
     </buffer>
 
-This configuration doesn't work with fluentd v0.12.
+See official article for more detail: [Buffer section configurations](http://docs.fluentd.org/v0.14/articles/buffer-section)
+
+Note that this configuration doesn't work with fluentd v0.12.
 
 ### v0.12 style
 
@@ -275,7 +280,7 @@ You can change key name by "message_key" option.
 
 Create S3 bucket if it does not exists. Default is true.
 
-**check_bukcet**
+**check_bucket**
 
 Check mentioned bucket if it exists in AWS or not. Default is true.
 
@@ -293,7 +298,7 @@ When it is false,
 	Example object name, assuming it is created on 2016/16/11 3:30:54 PM
 		20161611_153054.txt (extension can be anything as per user's choice)
 
-**Example when check_bukcet=false and check_object=false**
+**Example when check_bucket=false and check_object=false**
 
 When the mentioned configuration will be made, fluentd will work with the
 minimum IAM poilcy, like:

data/VERSION

@@ -1 +1 @@
-1.0.0.rc3
+1.0.0.rc4

data/lib/fluent/plugin/in_s3.rb

@@ -2,6 +2,7 @@ require 'fluent/plugin/input'
 require 'fluent/log-ext'
 
 require 'aws-sdk-resources'
+require 'cgi/util'
 require 'zlib'
 require 'time'
 require 'tempfile'
@@ -213,12 +214,12 @@ module Fluent::Plugin
       @bucket.objects.first
       log.debug("Succeeded to verify API keys")
     rescue => e
-      raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration. error = #{e.inspect}"
+      raise "can't call S3 API. Please check your credentials or s3_region configuration. error = #{e.inspect}"
     end
 
     def process(body)
       s3 = body["Records"].first["s3"]
-      key = s3["object"]["key"]
+      key = CGI.unescape(s3["object"]["key"])
 
       io = @bucket.object(key).get.body
       content = @extractor.extract(io)

data/lib/fluent/plugin/out_s3.rb

@@ -1,5 +1,6 @@
 require 'fluent/plugin/output'
 require 'fluent/log-ext'
+require 'fluent/timezone'
 require 'aws-sdk-resources'
 require 'zlib'
 require 'time'
@@ -157,6 +158,7 @@ module Fluent::Plugin
       # TODO: Remove time_slice_format when end of support compat_parameters
       @configured_time_slice_format = conf['time_slice_format']
       @values_for_s3_object_chunk = {}
+      @time_slice_with_tz = Fluent::Timezone.formatter(@timekey_zone, @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
     end
 
     def multi_workers_ready?
@@ -200,11 +202,10 @@ module Fluent::Plugin
       i = 0
       metadata = chunk.metadata
       previous_path = nil
-      time_slice_format = @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey'])
       time_slice = if metadata.timekey.nil?
                      ''.freeze
                    else
-                     Time.at(metadata.timekey).utc.strftime(time_slice_format)
+                     @time_slice_with_tz.call(metadata.timekey)
                    end
 
       if @check_object
@@ -248,6 +249,7 @@ module Fluent::Plugin
           "%{path}" => @path,
           "%{time_slice}" => time_slice,
           "%{file_extension}" => @compressor.ext,
+          "%{hms_slice}" => hms_slicer,
         }.merge!(@values_for_s3_object_chunk[chunk.unique_id])
         values_for_s3_object_key["%{uuid_flush}".freeze] = uuid_random if @uuid_flush_enabled
 
@@ -354,7 +356,7 @@ module Fluent::Plugin
     rescue Aws::S3::Errors::NoSuchBucket
       # ignore NoSuchBucket Error because ensure_bucket checks it.
     rescue => e
-      raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration. error = #{e.inspect}"
+      raise "can't call S3 API. Please check your credentials or s3_region configuration. error = #{e.inspect}"
     end
 
     def setup_credentials

data/test/test_in_s3.rb

@@ -185,6 +185,41 @@ class S3InputTest < Test::Unit::TestCase
     assert_equal({ "message" => "aaa" }, events.first[2])
   end
 
+  def test_one_record_url_encoded
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa") }
+    s3_object.get { s3_response }
+    @s3_bucket.object('test key').at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test+key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "message" => "aaa" }, events.first[2])
+  end
+
   def test_one_record_multi_line
     setup_mocks
     d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc3
+  version: 1.0.0.rc4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-02 00:00:00.000000000 Z
+date: 2017-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -155,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector