fluent-plugin-opensearch 1.1.6 → 1.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26499682224f8d7dc4a9b204f99172b512057fb5c9b67d03b0bf1637d39f4224
-  data.tar.gz: b6bc5ff55815d050f50a9c8658e98e0dbfce5511ade14342fa26d31c4dba59d0
+  metadata.gz: 59ef255b7e5cb5ceae9b97b0678cdc7151f0514263934f21e1028a45220180d6
+  data.tar.gz: 7603936854d8a36c74b6fa6ee4187b089cd781c997da8b97eed21fafbd52833c
 SHA512:
-  metadata.gz: a0ffa39d8a11088809008e5d74997d1765ca624153890bf0699ea2ab9f5522d80b2226427291bee42b04abf370636abd8a7dc5d8a13af1baf6cc4089e345da91
-  data.tar.gz: eb8983700b0a7af66010a5e8ec5f2f5ad5a30532fd877c7ab547aabfef015b45b012b83069c9775bb0965ebb95e01da8b75a8eb303d481ec768d076673679a3f
+  metadata.gz: 4caf0aa8ff2803a46536db82defff7b203f837dea3e2fc97b46487a04c9d85c5b4f71b9145985e9616e4db54f4bdef93a61ca85d941612474794276bdbf37531
+  data.tar.gz: b9fb24c499f3c84ece032e744a567ee9c34b7b8a2f47c39a706408e80fb7c8f19624499d447ed4191e75333c4050ef79237e51037dd0122ea95fc7aa6ad6f097

data/.github/dependabot.yml

@@ -4,3 +4,20 @@ updates:
     directory: '/'
     schedule:
       interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/add-to-project.yml

@@ -12,12 +12,12 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         with:
           app-id: ${{ secrets.PROJECT_APP_ID }}
           private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
           owner: fluent
-      - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
         with:
           project-url: https://github.com/orgs/fluent/projects/4
           github-token: ${{ steps.generate_token.outputs.token }}

data/.github/workflows/coverage.yaml

@@ -6,9 +6,9 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Ruby 3.0
-        uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
         with:
           ruby-version: '3.1'
       - name: Build and test with Rake
@@ -16,6 +16,6 @@ jobs:
           bundle install
           bundle exec rake
       - name: Coveralls GitHub Action
-        uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6
+        uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

data/.github/workflows/linux.yml

@@ -25,8 +25,8 @@ jobs:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/.github/workflows/macos.yml

@@ -25,8 +25,8 @@ jobs:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/.github/workflows/windows.yml

@@ -25,8 +25,8 @@ jobs:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/History.md

@@ -2,6 +2,13 @@
 
 ### [Unreleased]
 
+### 1.1.7
+ - test: remove duplicate keys from bulk response fixtures (#190)
+ - out\_opensearch: keep AWS credential providers refreshable to fix expired SigV4 tokens with assume\_role\_arn (#188)
+ - docs: document OpenSearch Serverless startup version-check workaround (#186)
+ - Allow duplicate keys in JSON parser (#185)
+ - Replace yajl-ruby with the json gem for JSON handling (#183)
+
 ### 1.1.6
  - out\_opensearch\_data\_stream: filter failed items in data stream bulk error log (#175)
  - out\_opensearch: fix default value of `refresh_credentials_interval` (#159)

data/README.md

@@ -902,7 +902,7 @@ compression_level best_compression
 
 ### prefer_oj_serializer
 
-With default behavior, OpenSearch client uses `Yajl` as JSON encoder/decoder.
+With default behavior, OpenSearch client uses `JSON` as JSON encoder/decoder.
 `Oj` is the alternative high performance JSON encoder/decoder.
 When this parameter sets as `true`, OpenSearch client uses `Oj` as JSON encoder/decoder.
 
@@ -1568,7 +1568,7 @@ In this case, the endpoint configuration looks like:
 
 ### Expiring AWS credentials
 
-If you want to expire AWS credentials in certain interval, you should specify `refresh_credentials_interval` parameter under `endpoint` section:
+Credentials obtained from STS, IMDS or ECS are refreshed by the AWS SDK before they expire, so you usually do not need to change this parameter. If you want to recreate the credential provider forcibly in a certain interval, you should specify `refresh_credentials_interval` parameter under `endpoint` section:
 
 ```aconf
 <endpoint>
@@ -1581,7 +1581,7 @@ If you want to expire AWS credentials in certain interval, you should specify `r
 
 ### Use OpenSearch Serverless
 
-If you want to use Serverless version of OpenSearch service, you have to specify `aoss` in `aws_service_name` under `endpoint` section:
+If you want to use the Serverless version of OpenSearch service, you have to specify `aoss` in `aws_service_name` under the `endpoint` section:
 
 ```aconf
 <endpoint>
@@ -1592,6 +1592,39 @@ If you want to use Serverless version of OpenSearch service, you have to specify
 </endpoint>
 ```
 
+**NOTE:** OpenSearch Serverless only supports a subset of the OpenSearch APIs and does **not** serve the root/info endpoint (`GET /`) that this plugin uses to detect the OpenSearch major version at startup. If you leave the default `verify_os_version_at_startup true`, the plugin fails the version probe with `404`, keeps retrying, and never starts sending data:
+
+```
+Could not communicate to OpenSearch, resetting connection and trying again. [404]
+```
+
+To avoid this, disable the startup version check and pin the version (Serverless is OpenSearch 2.x compatible):
+
+```aconf
+<match es.**>
+  @type opensearch
+  logstash_format true
+  include_tag_key true
+  flush_interval 1s
+
+  # Serverless does not serve GET /, so skip the version probe.
+  verify_os_version_at_startup false
+  default_opensearch_version 2
+
+  <endpoint>
+    url https://CLUSTER_ENDPOINT_URL
+    region us-east-2
+    aws_service_name aoss
+  </endpoint>
+</match>
+```
+
+**Other Serverless limitations to keep in mind:**
+
+- Your IAM identity needs a **data access policy** on the collection (at least `aoss:CreateIndex` and `aoss:WriteDocument`). An IAM permission policy alone is not sufficient.
+- For `TIMESERIES` collections, custom document IDs are not supported. Do not set `id_key`, and keep the default `write_operation index`, since `create` / `update` / `upsert` require an `_id`.
+- Index templates, rollover/ILM, and data streams rely on APIs that Serverless does not expose, so template-related options and `@type opensearch_data_stream` are not usable against a Serverless collection.
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.1.6'
+  s.version       = '1.1.7'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/in_opensearch.rb

@@ -374,7 +374,7 @@ module Fluent::Plugin
     def run_slice(slice_id=nil)
       slice_query = @base_query
       slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @num_slices}) unless slice_id.nil?
-      result = client.search(@options.merge(:body => Yajl.dump(slice_query) ))
+      result = client.search(@options.merge(:body => JSON.generate(slice_query) ))
       es = Fluent::MultiEventStream.new
 
       result["hits"]["hits"].each {|hit| process_events(hit, es)}

data/lib/fluent/plugin/opensearch_index_template.rb

@@ -33,7 +33,7 @@ module Fluent::OpenSearchIndexTemplate
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
-    JSON.parse(file_contents)
+    JSON.parse(file_contents, allow_duplicate_key: true)
   end
 
   def get_custom_template(template_file, customize_template)
@@ -44,7 +44,7 @@ module Fluent::OpenSearchIndexTemplate
     customize_template.each do |key, value|
       file_contents = file_contents.gsub(key,value.downcase)
     end
-    JSON.parse(file_contents)
+    JSON.parse(file_contents, allow_duplicate_key: true)
   end
 
   def template_exists?(name, host = nil)

data/lib/fluent/plugin/out_opensearch.rb

@@ -225,13 +225,13 @@ module Fluent::Plugin
         if conf[:assume_role_arn].nil?
           aws_container_credentials_relative_uri = conf[:ecs_container_credentials_relative_uri] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
           if aws_container_credentials_relative_uri.nil?
-            credentials = Aws::SharedCredentials.new({retries: 2}).credentials rescue nil
-            credentials ||= Aws::InstanceProfileCredentials.new.credentials rescue nil
-            credentials ||= Aws::ECSCredentials.new.credentials
+            credentials = usable_credentials_provider { Aws::SharedCredentials.new }
+            credentials ||= usable_credentials_provider { Aws::InstanceProfileCredentials.new }
+            credentials ||= usable_credentials_provider { Aws::ECSCredentials.new }
           else
             credentials = Aws::ECSCredentials.new({
                             credential_path: aws_container_credentials_relative_uri
-                          }).credentials
+                          })
           end
         else
           if conf[:assume_role_web_identity_token_file].nil?
@@ -239,21 +239,28 @@ module Fluent::Plugin
                             role_arn: conf[:assume_role_arn],
                             role_session_name: conf[:assume_role_session_name],
                             region: sts_creds_region(conf)
-                          }).credentials
+                          })
           else
             credentials = Aws::AssumeRoleWebIdentityCredentials.new({
                             role_arn: conf[:assume_role_arn],
                             web_identity_token_file: conf[:assume_role_web_identity_token_file],
                             region: sts_creds_region(conf)
-                          }).credentials
+                          })
           end
         end
       end
-      raise "No valid AWS credentials found." unless credentials.set?
+      raise "No valid AWS credentials found." unless credentials&.credentials&.set?
 
       credentials
     end
 
+    def usable_credentials_provider
+      provider = yield
+      provider if provider.credentials&.set?
+    rescue
+      nil
+    end
+
     def sts_creds_region(conf)
       conf[:sts_credentials_region] || conf[:region]
     end
@@ -339,7 +346,7 @@ module Fluent::Plugin
           }
         end
       end
-      # If AWS credentials is set, consider to expire credentials information forcibly before expired.
+      # Credential providers refresh themselves near expiration. This timer recreates them forcibly.
       @credential_mutex = Mutex.new
       if @endpoint
         @_aws_credentials = aws_credentials(@endpoint)
@@ -369,7 +376,7 @@ module Fluent::Plugin
           OpenSearch::API.settings[:serializer] = Fluent::Plugin::Serializer::Oj
         end
       rescue LoadError
-        @dump_proc = Yajl.method(:dump)
+        @dump_proc = JSON.method(:generate)
       end
 
       raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
@@ -630,7 +637,7 @@ module Fluent::Plugin
                              :aws_sigv4,
                              service: @endpoint.aws_service_name.to_s,
                              region: @endpoint.region,
-                             credentials: @_aws_credentials,
+                             credentials_provider: @_aws_credentials,
                            )
 
                            f.adapter @http_backend, @backend_options
@@ -940,7 +947,7 @@ module Fluent::Plugin
             {"_index" => {"order" => "desc"}}
          ]
         }
-        result = client.search(options.merge(:body => Yajl.dump(query)))
+        result = client.search(options.merge(:body => JSON.generate(query)))
         # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
         result['hits']['hits'].each do |hit|
           indices[hit["_id"]] = hit["_index"]

data/test/plugin/test_opensearch_error_handler.rb

@@ -596,7 +596,6 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
             "_type"  : "bar",
             "_id" : "6",
             "status" : 500,
-            "_type"  : "bar",
             "error" : {
               "type" : "out_of_memory_error",
               "reason":"Java heap space"

data/test/plugin/test_out_opensearch.rb

@@ -85,6 +85,27 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     }
   end
 
+  def chained_credentials_conf
+    {
+      access_key_id: '',
+      secret_access_key: '',
+      assume_role_arn: nil,
+      ecs_container_credentials_relative_uri: nil,
+    }
+  end
+
+  def with_stubbed_aws_responses
+    saved = Aws.config.key?(:stub_responses) ? Aws.config[:stub_responses] : :__unset__
+    Aws.config[:stub_responses] = true
+    yield
+  ensure
+    if saved == :__unset__
+      Aws.config.delete(:stub_responses)
+    else
+      Aws.config[:stub_responses] = saved
+    end
+  end
+
   def stub_opensearch_info(url="http://localhost:9200/", version="1.2.2")
     body ="{\"version\":{\"number\":\"#{version}\", \"distribution\":\"opensearch\"},\"tagline\":\"The OpenSearch Project: https://opensearch.org/\"}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
@@ -315,9 +336,106 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     assert_equal "fluentd", instance.endpoint.assume_role_session_name
     assert_nil instance.endpoint.assume_role_web_identity_token_file
     assert_nil instance.endpoint.sts_credentials_region
+    assert_equal 18000, instance.endpoint.refresh_credentials_interval
     assert_equal :es, instance.endpoint.aws_service_name
   end
 
+  test 'aws_credentials returns credential provider with access key' do
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+      }, [
+        Fluent::Config::Element.new('endpoint', '', {
+                                      'url' => "https://search-opensearch.aws.example.com/",
+                                      'region' => "local",
+                                      'access_key_id' => 'YOUR_AWESOME_KEY',
+                                      'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                    }, []),
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+
+      ])
+    instance = driver(config).instance
+    credentials = instance.aws_credentials(instance.endpoint)
+
+    assert_instance_of Aws::Credentials, credentials
+    assert_equal 'YOUR_AWESOME_KEY', credentials.credentials.access_key_id
+  end
+
+  test 'aws_credentials returns refreshable credential provider with assume_role_arn' do
+    with_stubbed_aws_responses do
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'opensearch',
+        }, [
+          Fluent::Config::Element.new('endpoint', '', {
+                                        'url' => "https://search-opensearch.aws.example.com/",
+                                        'region' => "local",
+                                        'assume_role_arn' => 'arn:aws:iam::123456789012:role/some-role',
+                                      }, []),
+          Fluent::Config::Element.new('buffer', 'tag', {}, [])
+
+        ])
+      instance = driver(config).instance
+      credentials = instance.aws_credentials(instance.endpoint)
+
+      assert_instance_of Aws::AssumeRoleCredentials, credentials
+      assert_kind_of Aws::RefreshingCredentials, credentials
+      assert_true credentials.credentials.set?
+    end
+  end
+
+  test 'aws_credentials falls back to the next provider when a provider has no credentials' do
+    unset_provider = flexmock('unset provider', credentials: Aws::Credentials.new(nil, nil))
+    ecs_provider = flexmock('ecs provider', credentials: Aws::Credentials.new('ECS_KEY', 'ECS_SECRET'))
+    flexmock(Aws::SharedCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::InstanceProfileCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::ECSCredentials).should_receive(:new).and_return(ecs_provider)
+    instance = driver.instance
+
+    assert_same ecs_provider, instance.aws_credentials(chained_credentials_conf)
+  end
+
+  test 'aws_credentials raises when no provider has usable credentials' do
+    unset_provider = flexmock('unset provider', credentials: Aws::Credentials.new(nil, nil))
+    flexmock(Aws::SharedCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::InstanceProfileCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::ECSCredentials).should_receive(:new)
+      .and_raise(ArgumentError.new("Cannot instantiate an ECS Credential Provider without a credential path or endpoint."))
+    instance = driver.instance
+
+    assert_raise(RuntimeError.new("No valid AWS credentials found.")) do
+      instance.aws_credentials(chained_credentials_conf)
+    end
+  end
+
+  test 'credentials provider is passed to aws_sigv4 middleware' do
+    with_stubbed_aws_responses do
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'opensearch',
+        }, [
+          Fluent::Config::Element.new('endpoint', '', {
+                                        'url' => "https://search-opensearch.aws.example.com/",
+                                        'region' => "local",
+                                        'assume_role_arn' => 'arn:aws:iam::123456789012:role/some-role',
+                                      }, []),
+          Fluent::Config::Element.new('buffer', 'tag', {}, [])
+
+        ])
+      instance = driver(config).instance
+      # Reaching into Faraday internals. If this breaks after a Faraday or
+      # faraday_middleware-aws-sigv4 update, check RackBuilder::Handler.
+      handler = instance.client.transport.transport.connections.first.connection.builder.handlers.find { |h|
+        h.klass == FaradayMiddleware::AwsSigV4
+      }
+
+      assert_not_nil handler
+      signer = handler.build(proc {}).instance_variable_get(:@signer)
+      assert_instance_of Aws::AssumeRoleCredentials, signer.credentials_provider
+      assert_same instance.instance_variable_get(:@_aws_credentials), signer.credentials_provider
+    end
+  end
+
   data("OpenSearch Service" => [:es, 'es'],
        "OpenSearch Serverless" => [:aoss, 'aoss'])
   test 'configure endpoint section w/ aws_service_name' do |data|
@@ -3648,7 +3766,6 @@ class OpenSearchOutputTest < Test::Unit::TestCase
                 "_index" : "foo",
                 "_type"  : "bar",
                 "_id" : "abc",
-                "_id" : "abc",
                 "status" : 409
               }
             }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.7
 platform: ruby
 authors:
 - Hiroshi Hatake
@@ -270,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector
 test_files: