fluent-plugin-s3 1.7.0 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db01fa7706627c40baf9e272644c4449c47e4adac7c67ae336099c1c65021e04
-  data.tar.gz: d74f9c06bcce4606b62d06ba416f934824213e42b508cf49dc093392c34692fe
+  metadata.gz: 1450176cd690d79412b7522ce8e67163c2f3c9c1f87941e57d6b3f3410bdf330
+  data.tar.gz: ee66229b8cd65ef2507feb5229d54322ed2f076e2daefb3f32e3b0e2ba47c634
 SHA512:
-  metadata.gz: 920c6ddc28b300bf9b00c2a21005d4adfeefc2e6c59ddb9afffedb2fcd45145baf70e2f7ec01997c12380d7b31def2289ece915a3099d0045b086314c914a555
-  data.tar.gz: eb84c4fc47ad9840fb375d65f532f7f5df701ce733b8959c6e037ffd45010548555ae5695b9e0839d4af60967ec791b5cb4cd049e103653b7b441a4f2bf54209
+  metadata.gz: f0428c1da0ef734a65d86aec878374636cf4bfd10ed2ab06578d3d8a9cc8ee607e62b4a85c05144599dec2167995dbd836b70cb7cfb66f38fc38d5e82dc1073a
+  data.tar.gz: bb4d7820ff8fa7fa3e58092b69fc78bb4804d92854206fb61c4efa21b415519eff682cf64763a9ec237f30f8997ab20b0b395bb8f1f17a8c88a9c63b3381b02b

data/ChangeLog

@@ -1,3 +1,13 @@
+Release 1.7.2 - 2022/10/19
+
+  * in_s3: Add `event_bridge_mode` parameter
+  * out_s3: Fix `s3_object_key_format` check to allow `%{hex_random}` as well as `%{uuid_flush}` or `${chunk_id}`
+
+Release 1.7.1 - 2022/07/15
+
+  * in_s3: Add `match_regexp` parameter to selectively download S3 files based on the object key
+  * out_s3: Support `ssl_ca_bundle` and `ssl_ca_directory` parameter
+
 Release 1.7.0 - 2022/06/14
 
   * in_s3: Allow multi workers

data/VERSION

@@ -1 +1 @@
-1.7.0
+1.7.2

data/docs/howto.md

@@ -7,7 +7,7 @@ downstream processors to better identify the source of a given record.
 
 # IAM Policy
 
-The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
+The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket/test, etc.).
 
     {
       "Version": "2012-10-17",

data/docs/input.md

@@ -18,6 +18,7 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
       s3_bucket YOUR_S3_BUCKET_NAME
       s3_region ap-northeast-1
       add_object_metadata true
+      match_regexp production_.*
 
       <sqs>
         queue_name YOUR_SQS_QUEUE_NAME
@@ -28,6 +29,10 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
 
 Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
 
+## match_regexp
+
+If provided, process the S3 object only if its keys matches the regular expression
+
 ## s3_bucket (required)
 
 S3 bucket name.
@@ -96,3 +101,7 @@ The long polling interval. Default is 20.
 ### retry_error_interval
 
 Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 300.
+
+### event_bridge_mode
+When true, Amazon S3 Event Notification should be configured using the EventBridge integration. Default is false.
+See [Configure S3 event notification using EventBridge](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) for additional information.

data/docs/output.md

@@ -81,6 +81,14 @@ This fixes the following error often seen in Windows:
 
     SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed (Seahorse::Client::NetworkingError)
 
+## ssl_ca_bundle
+
+Full path to the SSL certificate authority bundle file that should be used when verifying peer certificates. If you do not pass `ssl_ca_bundle` or `ssl_ca_directory` the the system default will be used if available.
+
+## ssl_ca_directory
+
+Full path of the directory that contains the unbundled SSL certificate authority files for verifying peer certificates. If you do not pass `ssl_ca_bundle` or `ssl_ca_directory` the the system default will be used if available.
+
 ## ssl_verify_peer
 
 Verify SSL certificate of the endpoint. Default is true. Set false when you want to ignore the endpoint SSL certificate.

data/lib/fluent/plugin/in_s3.rb

@@ -90,6 +90,8 @@ module Fluent::Plugin
     config_param :check_apikey_on_start, :bool, default: true
     desc "URI of proxy environment"
     config_param :proxy_uri, :string, default: nil
+    desc "Optional RegEx to match incoming messages"
+    config_param :match_regexp, :regexp, default: nil
 
     config_section :sqs, required: true, multi: false do
       desc "SQS queue name"
@@ -108,6 +110,8 @@ module Fluent::Plugin
       config_param :wait_time_seconds, :integer, default: 20
       desc "Polling error retry interval."
       config_param :retry_error_interval, :integer, default: 300
+      desc "Event bridge mode"
+      config_param :event_bridge_mode, :bool, default: false
     end
 
     desc "Tag string"
@@ -203,8 +207,12 @@ module Fluent::Plugin
           begin
             body = Yajl.load(message.body)
             log.debug(body)
-            next unless body["Records"] # skip test queue
-
+            next unless is_valid_queue(body) # skip test queue
+            if @match_regexp
+              raw_key = get_raw_key(body)
+              key = CGI.unescape(raw_key)
+              next unless @match_regexp.match?(key) 
+            end
             process(body)
           rescue => e
             log.warn(error: e)
@@ -219,6 +227,24 @@ module Fluent::Plugin
       end
     end
 
+    def is_valid_queue(body)
+      if @sqs.event_bridge_mode
+        log.debug("checking for eventbridge property")
+        !!body["detail"]
+      else 
+        log.debug("checking for Records property")
+        !!body["Records"]
+      end
+    end
+
+    def get_raw_key(body)
+      if @sqs.event_bridge_mode
+        body["detail"]["object"]["key"]
+      else
+        body["Records"].first["s3"]["object"]["key"]
+      end     
+    end
+
     def setup_credentials
       options = {}
       credentials_options = {}
@@ -311,8 +337,7 @@ module Fluent::Plugin
     end
 
     def process(body)
-      s3 = body["Records"].first["s3"]
-      raw_key = s3["object"]["key"]
+      raw_key = get_raw_key(body)
       key = CGI.unescape(raw_key)
 
       io = @bucket.object(key).get.body

data/lib/fluent/plugin/out_s3.rb

@@ -97,6 +97,10 @@ module Fluent::Plugin
     config_param :enable_dual_stack, :bool, default: false
     desc "If false, the certificate of endpoint will not be verified"
     config_param :ssl_verify_peer, :bool, :default => true
+    desc "Full path to the SSL certificate authority bundle file that should be used when verifying peer certificates. If unspecified, defaults to the system CA if available."
+    config_param :ssl_ca_bundle, :string, :default => nil
+    desc "Full path of the directory that contains the unbundled SSL certificate authority files for verifying peer certificates. If you do not pass ssl_ca_bundle or ssl_ca_directory the the system default will be used if available."
+    config_param :ssl_ca_directory, :string, :default => nil
     desc "The format of S3 object keys"
     config_param :s3_object_key_format, :string, default: "%{path}%{time_slice}_%{index}.%{file_extension}"
     desc "If true, the bucket name is always left in the request URI and never moved to the host as a sub-domain"
@@ -249,6 +253,8 @@ module Fluent::Plugin
       options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
       options[:signature_version] = @signature_version unless @signature_version.nil?
       options[:ssl_verify_peer] = @ssl_verify_peer
+      options[:ssl_ca_bundle] = @ssl_ca_bundle if @ssl_ca_bundle
+      options[:ssl_ca_directory] = @ssl_ca_directory if @ssl_ca_directory
       log.on_trace do
         options[:http_wire_trace] = true
         options[:logger] = log
@@ -465,8 +471,8 @@ module Fluent::Plugin
       end
 
       is_working_on_parallel = @buffer_config.flush_thread_count > 1 || system_config.workers > 1
-      if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
-        log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
+      if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}', '%{hex_random}'].none? { |key| @s3_object_key_format.include?(key) }
+        log.warn "No ${chunk_id}, %{uuid_flush} or %{hex_random} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id}, %{uuid_flush} or %{hex_random} to avoid data lost by object conflict"
       end
     end
 

data/test/test_in_s3.rb

@@ -166,9 +166,9 @@ buffer_type memory
   aws_key_id sqs_test_key_id
 </sqs>
 EOS
-        create_driver(conf)
-      }
-    end
+      create_driver(conf)
+    }
+  end
 
   def test_sqs_with_invalid_aws_keys_missing_key_id
     assert_raise(Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing") {
@@ -613,4 +613,114 @@ EOS
     ]
     assert_equal(expected_records, events.map {|_tag, _time, record| record })
   end
+
+  def test_regexp_matching
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp .*_key?")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa bbb ccc") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "message" => "aaa bbb ccc" }, events.first[2])
+  end
+
+  def test_regexp_not_matching
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp live?_key")
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    assert_nothing_raised do
+      d.run {}
+    end
+  end
+
+  def test_event_bridge_mode
+    setup_mocks
+    d = create_driver("
+      aws_key_id test_key_id
+      aws_sec_key test_sec_key
+      s3_bucket test_bucket
+      buffer_type memory
+      check_apikey_on_start false
+      store_as text
+      format none
+      <sqs>
+        event_bridge_mode true
+        queue_name test_queue
+        queue_owner_aws_account_id 123456789123
+      </sqs>
+    ")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "detail" => {
+        "object" => {
+          "key" => "test_key"
+        }
+      }
+    }
+    
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "message" => "aaa" }, events.first[2])
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-14 00:00:00.000000000 Z
+date: 2022-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd