fluent-plugin-s3 1.7.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db01fa7706627c40baf9e272644c4449c47e4adac7c67ae336099c1c65021e04
-  data.tar.gz: d74f9c06bcce4606b62d06ba416f934824213e42b508cf49dc093392c34692fe
+  metadata.gz: fe9afa0a2351dfffc6ea9afa0c2f3598a29b19a55fbfa22f58f3d17b606a3113
+  data.tar.gz: e5813910a178d16ed61c2a8782e2adeb394d0d256acfbcfdbead05069d789163
 SHA512:
-  metadata.gz: 920c6ddc28b300bf9b00c2a21005d4adfeefc2e6c59ddb9afffedb2fcd45145baf70e2f7ec01997c12380d7b31def2289ece915a3099d0045b086314c914a555
-  data.tar.gz: eb84c4fc47ad9840fb375d65f532f7f5df701ce733b8959c6e037ffd45010548555ae5695b9e0839d4af60967ec791b5cb4cd049e103653b7b441a4f2bf54209
+  metadata.gz: e3ca35df25975b7b57244c88d3145ab8fb5521054b863e030846a867bae6cbc9a69343f586f01b3a29d233eea787cf0ab540ff77edc0423d583394e29a192a7c
+  data.tar.gz: d46deabb1425e03250c9bc24578ad8dacef71cc445335ac9e4122b11c6ad91cf7a73f92b63d3688848ff35d427cb6371777f0011c94c45a898a6063edc1e3d9c

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 1.7.1 - 2022/07/15
+
+  * in_s3: Add `match_regexp` parameter to selectively download S3 files based on the object key
+  * out_s3: Support `ssl_ca_bundle` and `ssl_ca_directory` parameter
+
 Release 1.7.0 - 2022/06/14
 
   * in_s3: Allow multi workers

data/VERSION

@@ -1 +1 @@
-1.7.0
+1.7.1

data/docs/input.md

@@ -18,6 +18,7 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
       s3_bucket YOUR_S3_BUCKET_NAME
       s3_region ap-northeast-1
       add_object_metadata true
+      match_regexp production_.*
 
       <sqs>
         queue_name YOUR_SQS_QUEUE_NAME
@@ -28,6 +29,10 @@ See also [Configuration: credentials](credentials.md) for common comprehensive p
 
 Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
 
+## match_regexp
+
+If provided, process the S3 object only if its keys matches the regular expression
+
 ## s3_bucket (required)
 
 S3 bucket name.

data/docs/output.md

@@ -81,6 +81,14 @@ This fixes the following error often seen in Windows:
 
     SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed (Seahorse::Client::NetworkingError)
 
+## ssl_ca_bundle
+
+Full path to the SSL certificate authority bundle file that should be used when verifying peer certificates. If you do not pass `ssl_ca_bundle` or `ssl_ca_directory` the the system default will be used if available.
+
+## ssl_ca_directory
+
+Full path of the directory that contains the unbundled SSL certificate authority files for verifying peer certificates. If you do not pass `ssl_ca_bundle` or `ssl_ca_directory` the the system default will be used if available.
+
 ## ssl_verify_peer
 
 Verify SSL certificate of the endpoint. Default is true. Set false when you want to ignore the endpoint SSL certificate.

data/lib/fluent/plugin/in_s3.rb

@@ -90,6 +90,8 @@ module Fluent::Plugin
     config_param :check_apikey_on_start, :bool, default: true
     desc "URI of proxy environment"
     config_param :proxy_uri, :string, default: nil
+    desc "Optional RegEx to match incoming messages"
+    config_param :match_regexp, :regexp, default: nil
 
     config_section :sqs, required: true, multi: false do
       desc "SQS queue name"
@@ -204,7 +206,12 @@ module Fluent::Plugin
             body = Yajl.load(message.body)
             log.debug(body)
             next unless body["Records"] # skip test queue
-
+            if @match_regexp
+              s3 = body["Records"].first["s3"]
+              raw_key = s3["object"]["key"]
+              key = CGI.unescape(raw_key)
+              next unless @match_regexp.match?(key) 
+            end
             process(body)
           rescue => e
             log.warn(error: e)

data/lib/fluent/plugin/out_s3.rb

@@ -97,6 +97,10 @@ module Fluent::Plugin
     config_param :enable_dual_stack, :bool, default: false
     desc "If false, the certificate of endpoint will not be verified"
     config_param :ssl_verify_peer, :bool, :default => true
+    desc "Full path to the SSL certificate authority bundle file that should be used when verifying peer certificates. If unspecified, defaults to the system CA if available."
+    config_param :ssl_ca_bundle, :string, :default => nil
+    desc "Full path of the directory that contains the unbundled SSL certificate authority files for verifying peer certificates. If you do not pass ssl_ca_bundle or ssl_ca_directory the the system default will be used if available."
+    config_param :ssl_ca_directory, :string, :default => nil
     desc "The format of S3 object keys"
     config_param :s3_object_key_format, :string, default: "%{path}%{time_slice}_%{index}.%{file_extension}"
     desc "If true, the bucket name is always left in the request URI and never moved to the host as a sub-domain"
@@ -249,6 +253,8 @@ module Fluent::Plugin
       options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
       options[:signature_version] = @signature_version unless @signature_version.nil?
       options[:ssl_verify_peer] = @ssl_verify_peer
+      options[:ssl_ca_bundle] = @ssl_ca_bundle if @ssl_ca_bundle
+      options[:ssl_ca_directory] = @ssl_ca_directory if @ssl_ca_directory
       log.on_trace do
         options[:http_wire_trace] = true
         options[:logger] = log

data/test/test_in_s3.rb

@@ -613,4 +613,68 @@ EOS
     ]
     assert_equal(expected_records, events.map {|_tag, _time, record| record })
   end
+
+  def test_regexp_matching
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp .*_key?")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa bbb ccc") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "message" => "aaa bbb ccc" }, events.first[2])
+  end
+
+  def test_regexp_not_matching
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nmatch_regexp live?_key")
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    assert_nothing_raised do
+      d.run {}
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-14 00:00:00.000000000 Z
+date: 2022-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd