fluent-plugin-s3 1.1.5 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73a256aab05c981aeb3122a36caaaae7c878e915
-  data.tar.gz: 665d1edb67fd09763207d8c9be3c5dde7fdaac38
+  metadata.gz: 0263bee7f101a3d938fb85697b7acf0473b9f3ed
+  data.tar.gz: 1f7b330dc029d10f50f9eee56a1f2310ebcca839
 SHA512:
-  metadata.gz: 59e0391e4e3cf9235cdc6aed4917aad1e6717afd6c88cbc076bb5b8bb888d03ad0e1e27e413da589131e7b34e25b22be47cc3d78b7a1d98b5e8d2e4a88cd8786
-  data.tar.gz: 0732a187a568c066f914d551337d4d0fc27cd71f1e31ed4bc9728d692648c1bdb060dc762ce53e7c0d0e8102392645876805755e69b98b30e634a9c44b5b9294
+  metadata.gz: 1fafe6a2e4c311f4850afa2f241332292f31849c167010bd187f9a3cddcce03308bd41a8fb01a9447e3c86ae181d67857a28f8a26b441da4b2203019443ef95a
+  data.tar.gz: ad7e122c740eb2dbeaf17d39927b2ecb4474e60bc81e2d9592647568565fb453821607a4e8ebc62f940186fc68c7d4e3c5fd47a853123e56ec17235fc4f6003b

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.1.6 - 2018/09/11
+
+  * in_s3: Add s3_endpoint parameter to support S3 compatible service
+
 Release 1.1.5 - 2018/09/04
 
   * out_s3: Improve check_apikeys performance by specifying `max_keys` parameter

data/VERSION

@@ -1 +1 @@
-1.1.5
+1.1.6

data/lib/fluent/plugin/in_s3.rb

@@ -64,6 +64,10 @@ module Fluent::Plugin
     config_param :s3_bucket, :string
     desc "S3 region name"
     config_param :s3_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
+    desc "Use 's3_region' instead"
+    config_param :s3_endpoint, :string, default: nil
+    desc "If true, the bucket name is always left in the request URI and never moved to the host as a sub-domain"
+    config_param :force_path_style, :bool, default: false
     desc "Archive format on S3"
     config_param :store_as, :string, default: "gzip"
     desc "Check AWS key on start"
@@ -74,6 +78,8 @@ module Fluent::Plugin
     config_section :sqs, required: true, multi: false do
       desc "SQS queue name"
       config_param :queue_name, :string, default: nil
+      desc "Use 's3_region' instead"
+      config_param :endpoint, :string, default: nil
       desc "Skip message deletion"
       config_param :skip_delete, :bool, default: false
       desc "The long polling interval."
@@ -92,6 +98,14 @@ module Fluent::Plugin
     def configure(conf)
       super
 
+      if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
+        raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
+      end
+
+      if @sqs.endpoint && @sqs.endpoint.end_with?('amazonaws.com')
+        raise Fluent::ConfigError, "sqs/endpoint parameter is not supported for SQS, use s3_region instead. This parameter is for SQS compatible services"
+      end
+
       parser_config = conf.elements("parse").first
       unless @sqs.queue_name
         raise Fluent::ConfigError, "sqs/queue_name is required"
@@ -196,6 +210,8 @@ module Fluent::Plugin
     def create_s3_client
       options = setup_credentials
       options[:region] = @s3_region if @s3_region
+      options[:endpoint] = @s3_endpoint if @s3_endpoint
+      options[:force_path_style] = @force_path_style
       options[:proxy_uri] = @proxy_uri if @proxy_uri
       log.on_trace do
         options[:http_wire_trace] = true
@@ -208,6 +224,7 @@ module Fluent::Plugin
     def create_sqs_client
       options = setup_credentials
       options[:region] = @s3_region if @s3_region
+      options[:endpoint] = @sqs.endpoint if @sqs.endpoint
       log.on_trace do
         options[:http_wire_trace] = true
         options[:logger] = log

data/test/test_in_s3.rb

@@ -112,6 +112,40 @@ class S3InputTest < Test::Unit::TestCase
     end
   end
 
+
+  def test_s3_endpoint_with_valid_endpoint
+    d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
+    assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
+  end
+
+  data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
+       'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
+       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+  def test_s3_endpoint_with_invalid_endpoint(endpoint)
+    assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
+      create_driver(CONFIG + "s3_endpoint #{endpoint}")
+    }
+  end
+
+  data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
+       'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
+       'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
+  def test_sqs_endpoint_with_invalid_endpoint(endpoint)
+    assert_raise(Fluent::ConfigError, "sqs.endpoint parameter is not supported, use s3_region instead. This parameter is for SQS compatible services") {
+      conf = <<"EOS"
+aws_key_id test_key_id
+aws_sec_key test_sec_key
+s3_bucket test_bucket
+buffer_type memory
+<sqs>
+  queue_name test_queue
+  endpoint #{endpoint}
+</sqs>
+EOS
+      create_driver(conf)
+    }
+  end
+
   Struct.new("StubResponse", :queue_url)
   Struct.new("StubMessage", :message_id, :receipt_handle, :body)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.1.5
+  version: 1.1.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-05 00:00:00.000000000 Z
+date: 2018-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd