fluent-plugin-s3 1.1.10 → 1.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f75104b3bef77164ce2e595b52615189a7e71aa5
-  data.tar.gz: cbaf573a501196ce198140f01d836401b1a83ca6
+SHA256:
+  metadata.gz: e034c8d5e4ae06a88f405bfe396d2c1c960da096b5010d4b45902765b00eaa6a
+  data.tar.gz: 679865ce71becff176745c0fcb051a6120313c5127d4ad70a8333b263c5eb67d
 SHA512:
-  metadata.gz: 10f7e05421fd2fbf5b9e33c8d2103dd0b7c8f19a45f362e2116d181cddb93d19f4733bbc10fb16ca6647352db684fe1a92ffb081e04964db57d9526c2cb28c0f
-  data.tar.gz: 42bfe6daab63ded2bf6f4800d6c3322d653d9290c7e7ba3d8f65d93b86576e0d9a5850410b5c403abd4af90e9c4dc4f90280b45c2c08ef769b26b769ac223978
+  metadata.gz: b0e0bf4a2072b9589e0e27ed84806c00d1fcc1ba47e70d1a628017aa7981993372b2afd7ee04d1eee5c558c2d485c3128ec64318b34fd802b8c1759fa311d71d
+  data.tar.gz: 6fd057fd51ce0aeed1fd64c2a39a405ee329f0b96cb92dc09faeac37f4d679fc0d7951d827f8ea0b0eda513a99f9cd89ab171b795933b1d8a44294d025fac7c8

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.1.11 - 2019/06/17
+
+  * in_s3: Add add_object_metadata parameter
+
 Release 1.1.10 - 2019/05/08
 
   * out_s3: force_path_style is deprecated

data/README.md

@@ -202,7 +202,7 @@ For `<buffer>`, you can use any record field in `path` / `s3_object_key_format`.
       # parameters...
     </buffer>
 
-See official article for more detail: [Buffer section configurations](http://docs.fluentd.org/v1.0/articles/buffer-section)
+See official article for more detail: [Config: Buffer Section - Fluentd](https://docs.fluentd.org/configuration/buffer-section)
 
 Note that this configuration doesn't work with fluentd v0.12.
 
@@ -322,7 +322,7 @@ You get:
     "log/events/ts=20130111-23/events_1.gz"
     "log/events/ts=20130112-00/events_0.gz"
 
-NOTE: ${hostname} placeholder is deprecated since v0.8. You can get same result by using [configuration's embedded ruby code feature](http://docs.fluentd.org/articles/config-file#embedded-ruby-code).
+NOTE: ${hostname} placeholder is deprecated since v0.8. You can get same result by using [configuration's embedded ruby code feature](https://docs.fluentd.org/configuration/config-file#embedded-ruby-code).
 
     s3_object_key_format %{path}%{time_slice}_%{hostname}%{index}.%{file_extension}
     s3_object_key_format "%{path}%{time_slice}_#{Socket.gethostname}%{index}.%{file_extension}"
@@ -355,7 +355,7 @@ See `Use your compression algorithm` section for adding another format.
 **`<format>` or format**
 
 Change one line format in the S3 object. Supported formats are "out_file",
-"json", "ltsv" and "single_value". See also [official Formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
+"json", "ltsv" and "single_value". See also [official Formatter article](https://docs.fluentd.org/formatter).
 
 * out_file (default).
 
@@ -586,12 +586,17 @@ Given a threshold to treat events as delay, output warning logs if delayed event
       aws_sec_key YOUR_AWS_SECRET_KEY
       s3_bucket YOUR_S3_BUCKET_NAME
       s3_region ap-northeast-1
+      add_object_metadata true
 
       <sqs>
         queue_name YOUR_SQS_QUEUE_NAME
       </sqs>
     </source>
 
+**add_object_metadata**
+
+Whether or not object metadata should be added to the record. Defaults to `false`. See below for details.
+
 **s3_bucket (required)**
 
 S3 bucket name.
@@ -647,6 +652,13 @@ The long polling interval. Default is 20.
 
 Interval to retry polling SQS if polling unsuccessful, in seconds. Default is 300.
 
+## Object Metadata Added To Records
+
+If the `add_object_metadata` option is set to true, then the name of the bucket
+and the key for a given object will be added to each log record as `s3_bucket`
+and `s3_key`, respectively. This metadata can be used by filter plugins or other
+downstream processors to better identify the source of a given record.
+
 ## IAM Policy
 
 The following is an example for a IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).

data/VERSION

@@ -1 +1 @@
-1.1.10
+1.1.11

data/fluent-plugin-s3.gemspec

@@ -10,7 +10,6 @@ Gem::Specification.new do |gem|
   gem.version     = File.read("VERSION").strip
   gem.authors     = ["Sadayuki Furuhashi", "Masahiro Nakagawa"]
   gem.email       = "frsyuki@gmail.com"
-  gem.has_rdoc    = false
   #gem.platform    = Gem::Platform::RUBY
   gem.files       = `git ls-files`.split("\n")
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/in_s3.rb

@@ -24,6 +24,8 @@ module Fluent::Plugin
 
     desc "Use aws-sdk-ruby bundled cert"
     config_param :use_bundled_cert, :bool, default: false
+    desc "Add object metadata to the records parsed out of a given object"
+    config_param :add_object_metadata, :bool, default: false
     desc "AWS access key id"
     config_param :aws_key_id, :string, default: nil, secret: true
     desc "AWS secret key."
@@ -256,13 +258,18 @@ module Fluent::Plugin
 
     def process(body)
       s3 = body["Records"].first["s3"]
-      key = CGI.unescape(s3["object"]["key"])
+      raw_key = s3["object"]["key"]
+      key = CGI.unescape(raw_key)
 
       io = @bucket.object(key).get.body
       content = @extractor.extract(io)
       es = Fluent::MultiEventStream.new
       content.each_line do |line|
         @parser.parse(line) do |time, record|
+          if @add_object_metadata
+            record['s3_bucket'] = @s3_bucket
+            record['s3_key'] = raw_key
+          end
           es.add(time, record)
         end
       end

data/test/test_in_s3.rb

@@ -224,6 +224,41 @@ EOS
     assert_equal({ "message" => "aaa" }, events.first[2])
   end
 
+  def test_one_record_with_metadata
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nadd_object_metadata true\n")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "aaa" }, events.first[2])
+  end
+
   def test_one_record_url_encoded
     setup_mocks
     d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n")
@@ -259,6 +294,41 @@ EOS
     assert_equal({ "message" => "aaa" }, events.first[2])
   end
 
+  def test_one_record_url_encoded_with_metadata
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nadd_object_metadata true")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa") }
+    s3_object.get { s3_response }
+    @s3_bucket.object('test key').at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test+key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    assert_equal({ "s3_bucket" => "test_bucket", "s3_key" => "test+key", "message" => "aaa" }, events.first[2])
+  end
+
   def test_one_record_multi_line
     setup_mocks
     d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\n")
@@ -299,6 +369,46 @@ EOS
     assert_equal(expected_records, events.map {|_tag, _time, record| record })
   end
 
+  def test_one_record_multi_line_with_metadata
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as text\nformat none\nadd_object_metadata true")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body { StringIO.new("aaa\nbbb\nccc\n") }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    expected_records = [
+      { "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "aaa\n" },
+      { "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "bbb\n" },
+      { "s3_bucket" => "test_bucket", "s3_key" => "test_key", "message" => "ccc\n" }
+    ]
+    assert_equal(expected_records, events.map {|_tag, _time, record| record })
+  end
+
   def test_gzip_single_stream
     setup_mocks
     d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as gzip\nformat none\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.1.10
+  version: 1.1.11
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-09 00:00:00.000000000 Z
+date: 2019-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -162,8 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector