fluent-plugin-s3 1.7.1 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe9afa0a2351dfffc6ea9afa0c2f3598a29b19a55fbfa22f58f3d17b606a3113
-  data.tar.gz: e5813910a178d16ed61c2a8782e2adeb394d0d256acfbcfdbead05069d789163
+  metadata.gz: 1450176cd690d79412b7522ce8e67163c2f3c9c1f87941e57d6b3f3410bdf330
+  data.tar.gz: ee66229b8cd65ef2507feb5229d54322ed2f076e2daefb3f32e3b0e2ba47c634
 SHA512:
-  metadata.gz: e3ca35df25975b7b57244c88d3145ab8fb5521054b863e030846a867bae6cbc9a69343f586f01b3a29d233eea787cf0ab540ff77edc0423d583394e29a192a7c
-  data.tar.gz: d46deabb1425e03250c9bc24578ad8dacef71cc445335ac9e4122b11c6ad91cf7a73f92b63d3688848ff35d427cb6371777f0011c94c45a898a6063edc1e3d9c
+  metadata.gz: f0428c1da0ef734a65d86aec878374636cf4bfd10ed2ab06578d3d8a9cc8ee607e62b4a85c05144599dec2167995dbd836b70cb7cfb66f38fc38d5e82dc1073a
+  data.tar.gz: bb4d7820ff8fa7fa3e58092b69fc78bb4804d92854206fb61c4efa21b415519eff682cf64763a9ec237f30f8997ab20b0b395bb8f1f17a8c88a9c63b3381b02b

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 1.7.2 - 2022/10/19
+
+  * in_s3: Add `event_bridge_mode` parameter
+  * out_s3: Fix `s3_object_key_format` check to allow `%{hex_random}` as well as `%{uuid_flush}` or `${chunk_id}`
+
 Release 1.7.1 - 2022/07/15
 
   * in_s3: Add `match_regexp` parameter to selectively download S3 files based on the object key

data/VERSION

@@ -1 +1 @@
-1.7.1
+1.7.2

data/docs/howto.md

@@ -7,7 +7,7 @@ downstream processors to better identify the source of a given record.
 
 # IAM Policy
 
-The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
+The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket/test, etc.).
 
     {
       "Version": "2012-10-17",

data/docs/input.md

@@ -101,3 +101,7 @@ The long polling interval. Default is 20.
 ### retry_error_interval
 
 Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 300.
+
+### event_bridge_mode
+When true, Amazon S3 Event Notification should be configured using the EventBridge integration. Default is false.
+See [Configure S3 event notification using EventBridge](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) for additional information.

data/lib/fluent/plugin/in_s3.rb

@@ -110,6 +110,8 @@ module Fluent::Plugin
       config_param :wait_time_seconds, :integer, default: 20
       desc "Polling error retry interval."
       config_param :retry_error_interval, :integer, default: 300
+      desc "Event bridge mode"
+      config_param :event_bridge_mode, :bool, default: false
     end
 
     desc "Tag string"
@@ -205,10 +207,9 @@ module Fluent::Plugin
           begin
             body = Yajl.load(message.body)
             log.debug(body)
-            next unless body["Records"] # skip test queue
+            next unless is_valid_queue(body) # skip test queue
             if @match_regexp
-              s3 = body["Records"].first["s3"]
-              raw_key = s3["object"]["key"]
+              raw_key = get_raw_key(body)
               key = CGI.unescape(raw_key)
               next unless @match_regexp.match?(key) 
             end
@@ -226,6 +227,24 @@ module Fluent::Plugin
       end
     end
 
+    def is_valid_queue(body)
+      if @sqs.event_bridge_mode
+        log.debug("checking for eventbridge property")
+        !!body["detail"]
+      else 
+        log.debug("checking for Records property")
+        !!body["Records"]
+      end
+    end
+
+    def get_raw_key(body)
+      if @sqs.event_bridge_mode
+        body["detail"]["object"]["key"]
+      else
+        body["Records"].first["s3"]["object"]["key"]
+      end     
+    end
+
     def setup_credentials
       options = {}
       credentials_options = {}
@@ -318,8 +337,7 @@ module Fluent::Plugin
     end
 
     def process(body)
-      s3 = body["Records"].first["s3"]
-      raw_key = s3["object"]["key"]
+      raw_key = get_raw_key(body)
       key = CGI.unescape(raw_key)
 
       io = @bucket.object(key).get.body

data/lib/fluent/plugin/out_s3.rb

@@ -471,8 +471,8 @@ module Fluent::Plugin
       end
 
       is_working_on_parallel = @buffer_config.flush_thread_count > 1 || system_config.workers > 1
-      if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
-        log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
+      if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}', '%{hex_random}'].none? { |key| @s3_object_key_format.include?(key) }
+        log.warn "No ${chunk_id}, %{uuid_flush} or %{hex_random} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id}, %{uuid_flush} or %{hex_random} to avoid data lost by object conflict"
       end
     end
 

data/test/test_in_s3.rb

@@ -166,9 +166,9 @@ buffer_type memory
   aws_key_id sqs_test_key_id
 </sqs>
 EOS
-        create_driver(conf)
-      }
-    end
+      create_driver(conf)
+    }
+  end
 
   def test_sqs_with_invalid_aws_keys_missing_key_id
     assert_raise(Fluent::ConfigError, "sqs/aws_key_id or sqs/aws_sec_key is missing") {
@@ -677,4 +677,50 @@ EOS
       d.run {}
     end
   end
+
+  def test_event_bridge_mode
+    setup_mocks
+    d = create_driver("
+      aws_key_id test_key_id
+      aws_sec_key test_sec_key
+      s3_bucket test_bucket
+      buffer_type memory
+      check_apikey_on_start false
+      store_as text
+      format none
+      <sqs>
+        event_bridge_mode true
+        queue_name test_queue
+        queue_owner_aws_account_id 123456789123
+      </sqs>
+    ")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "detail" => {
+        "object" => {
+          "key" => "test_key"
+        }
+      }
+    }
+    
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "message" => "aaa" }, events.first[2])
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 2022-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd