fluent-plugin-opensearch 1.1.6 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 26499682224f8d7dc4a9b204f99172b512057fb5c9b67d03b0bf1637d39f4224
-  data.tar.gz: b6bc5ff55815d050f50a9c8658e98e0dbfce5511ade14342fa26d31c4dba59d0
+  metadata.gz: '04843ec3381adb39f0e7fbc2a6cc76c25fa8d1d85d5733e500e58d7814beb57a'
+  data.tar.gz: b5e60e70521053e723b9c4df9953b03774a1207f7cf3c19b9c3aeb385e03f15f
 SHA512:
-  metadata.gz: a0ffa39d8a11088809008e5d74997d1765ca624153890bf0699ea2ab9f5522d80b2226427291bee42b04abf370636abd8a7dc5d8a13af1baf6cc4089e345da91
-  data.tar.gz: eb8983700b0a7af66010a5e8ec5f2f5ad5a30532fd877c7ab547aabfef015b45b012b83069c9775bb0965ebb95e01da8b75a8eb303d481ec768d076673679a3f
+  metadata.gz: 80cb6808fef85cc16fb835324f2ff3fbe3d42978d3c3f72dccf35251d39b3fe29f74d1271c30b2913959fd7cdc1f4e277ed91608663756e24d3d2e9b438e2943
+  data.tar.gz: 6d936e5608366a827cdd03f8bf51615b34780f9fdb7354a764cb2f39e4c570ad5f768ccc73cf286c9f22a3bafc529868b7e4434c8ea6b001af3328c541a7cb3e

data/.github/dependabot.yml

@@ -4,3 +4,20 @@ updates:
     directory: '/'
     schedule:
       interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/add-to-project.yml

@@ -12,12 +12,12 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         with:
           app-id: ${{ secrets.PROJECT_APP_ID }}
           private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
           owner: fluent
-      - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
         with:
           project-url: https://github.com/orgs/fluent/projects/4
           github-token: ${{ steps.generate_token.outputs.token }}

data/.github/workflows/coverage.yaml

@@ -6,9 +6,9 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Ruby 3.0
-        uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+        uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
         with:
           ruby-version: '3.1'
       - name: Build and test with Rake
@@ -16,6 +16,6 @@ jobs:
           bundle install
           bundle exec rake
       - name: Coveralls GitHub Action
-        uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6
+        uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

data/.github/workflows/linux.yml

@@ -25,8 +25,8 @@ jobs:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/.github/workflows/macos.yml

@@ -25,8 +25,8 @@ jobs:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/.github/workflows/windows.yml

@@ -25,8 +25,8 @@ jobs:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/History.md

@@ -2,6 +2,18 @@
 
 ### [Unreleased]
 
+### 1.1.8
+ - `out_opensearch`: limit what a host placeholder can expand to (#192)
+ - `out_opensearch`: escape a `customize_template` value inside a string (#194)
+ - `out_opensearch`: lock the client cache against flush threads (#195)
+
+### 1.1.7
+ - test: remove duplicate keys from bulk response fixtures (#190)
+ - out\_opensearch: keep AWS credential providers refreshable to fix expired SigV4 tokens with assume\_role\_arn (#188)
+ - docs: document OpenSearch Serverless startup version-check workaround (#186)
+ - Allow duplicate keys in JSON parser (#185)
+ - Replace yajl-ruby with the json gem for JSON handling (#183)
+
 ### 1.1.6
  - out\_opensearch\_data\_stream: filter failed items in data stream bulk error log (#175)
  - out\_opensearch: fix default value of `refresh_credentials_interval` (#159)

data/README.md

@@ -902,7 +902,7 @@ compression_level best_compression
 
 ### prefer_oj_serializer
 
-With default behavior, OpenSearch client uses `Yajl` as JSON encoder/decoder.
+With default behavior, OpenSearch client uses `JSON` as JSON encoder/decoder.
 `Oj` is the alternative high performance JSON encoder/decoder.
 When this parameter sets as `true`, OpenSearch client uses `Oj` as JSON encoder/decoder.
 
@@ -1241,6 +1241,12 @@ records = {key1: "value1", key2: "value2"}
 </match>
 ```
 
+#### host and hosts
+
+`host` and `hosts` take placeholders too, for example `hosts https://logs-${tag}.example.com:9201/os`. A value may only fill in the part of a host name that the placeholder stands for, and how much it may fill in depends on the names that you write after the placeholder. `logs-${tag}.example.com` lets a value add a sub domain of `example.com`, such as `nginx.access`. `logs-${tag}.com` lets a value fill in one name only, because a single name fixes the top level domain and nothing else: the name that decides who owns the domain would still come from the value. A value that does not fit raises `UnrecoverableRequestFailure`.
+
+**Note**: the configured `user`, `password` and `custom_headers` are sent to whichever host the value picks. So write a domain that you control, where nobody else can create a sub domain. A shared domain such as `amazonaws.com` is not enough. Without such a domain, let only senders that you trust set the placeholder value.
+
 ## Multi workers
 
 Since Fluentd v0.14, multi workers feature had been implemented to increase throughput with multiple processes. This feature allows Fluentd processes to use one or more CPUs. This feature will be enabled by the following system configuration:
@@ -1568,7 +1574,7 @@ In this case, the endpoint configuration looks like:
 
 ### Expiring AWS credentials
 
-If you want to expire AWS credentials in certain interval, you should specify `refresh_credentials_interval` parameter under `endpoint` section:
+Credentials obtained from STS, IMDS or ECS are refreshed by the AWS SDK before they expire, so you usually do not need to change this parameter. If you want to recreate the credential provider forcibly in a certain interval, you should specify `refresh_credentials_interval` parameter under `endpoint` section:
 
 ```aconf
 <endpoint>
@@ -1581,7 +1587,7 @@ If you want to expire AWS credentials in certain interval, you should specify `r
 
 ### Use OpenSearch Serverless
 
-If you want to use Serverless version of OpenSearch service, you have to specify `aoss` in `aws_service_name` under `endpoint` section:
+If you want to use the Serverless version of OpenSearch service, you have to specify `aoss` in `aws_service_name` under the `endpoint` section:
 
 ```aconf
 <endpoint>
@@ -1592,6 +1598,39 @@ If you want to use Serverless version of OpenSearch service, you have to specify
 </endpoint>
 ```
 
+**NOTE:** OpenSearch Serverless only supports a subset of the OpenSearch APIs and does **not** serve the root/info endpoint (`GET /`) that this plugin uses to detect the OpenSearch major version at startup. If you leave the default `verify_os_version_at_startup true`, the plugin fails the version probe with `404`, keeps retrying, and never starts sending data:
+
+```
+Could not communicate to OpenSearch, resetting connection and trying again. [404]
+```
+
+To avoid this, disable the startup version check and pin the version (Serverless is OpenSearch 2.x compatible):
+
+```aconf
+<match es.**>
+  @type opensearch
+  logstash_format true
+  include_tag_key true
+  flush_interval 1s
+
+  # Serverless does not serve GET /, so skip the version probe.
+  verify_os_version_at_startup false
+  default_opensearch_version 2
+
+  <endpoint>
+    url https://CLUSTER_ENDPOINT_URL
+    region us-east-2
+    aws_service_name aoss
+  </endpoint>
+</match>
+```
+
+**Other Serverless limitations to keep in mind:**
+
+- Your IAM identity needs a **data access policy** on the collection (at least `aoss:CreateIndex` and `aoss:WriteDocument`). An IAM permission policy alone is not sufficient.
+- For `TIMESERIES` collections, custom document IDs are not supported. Do not set `id_key`, and keep the default `write_operation index`, since `create` / `update` / `upsert` require an `_id`.
+- Index templates, rollover/ILM, and data streams rely on APIs that Serverless does not expose, so template-related options and `@type opensearch_data_stream` are not usable against a Serverless collection.
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.1.6'
+  s.version       = '1.1.8'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/in_opensearch.rb

@@ -51,7 +51,9 @@ module Fluent::Plugin
     config_param :password, :string, :default => nil, :secret => true
     config_param :path, :string, :default => nil
     config_param :scheme, :enum, :list => [:https, :http], :default => :http
-    config_param :hosts, :string, :default => nil
+    # `hosts` accepts "https://user:password@host", so it is masked in the
+    # configuration dump like `password` is.
+    config_param :hosts, :string, :default => nil, :secret => true
     config_param :index_name, :string, :default => "fluentd"
     config_param :parse_timestamp, :bool, :default => false
     config_param :timestamp_key_format, :string, :default => nil
@@ -374,7 +376,7 @@ module Fluent::Plugin
     def run_slice(slice_id=nil)
       slice_query = @base_query
       slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @num_slices}) unless slice_id.nil?
-      result = client.search(@options.merge(:body => Yajl.dump(slice_query) ))
+      result = client.search(@options.merge(:body => JSON.generate(slice_query) ))
       es = Fluent::MultiEventStream.new
 
       result["hits"]["hits"].each {|hit| process_events(hit, es)}

data/lib/fluent/plugin/opensearch_index_template.rb

@@ -24,6 +24,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require 'json'
+
 require 'fluent/error'
 require_relative './opensearch_error'
 
@@ -33,7 +35,7 @@ module Fluent::OpenSearchIndexTemplate
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
-    JSON.parse(file_contents)
+    JSON.parse(file_contents, allow_duplicate_key: true)
   end
 
   def get_custom_template(template_file, customize_template)
@@ -42,9 +44,23 @@ module Fluent::OpenSearchIndexTemplate
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
     customize_template.each do |key, value|
-      file_contents = file_contents.gsub(key,value.downcase)
+      file_contents = substitute_template_value(file_contents, key, value.downcase)
+    end
+    JSON.parse(file_contents, allow_duplicate_key: true)
+  end
+
+  # A value can come from a tag or a record field. If its key is inside a JSON
+  # string ($1), a `"` in the value would end that string, and the rest would
+  # be read as extra template keys, so the value is escaped there. If the key
+  # is anywhere else, the value is a piece of JSON, like a whole settings
+  # object, so it goes in as it is. The block form stops gsub from reading
+  # backslash sequences in the value.
+  def substitute_template_value(file_contents, key, value)
+    json_string = /"(?:\\.|[^"\\])*"/
+    file_contents.gsub(/(#{json_string})|#{Regexp.escape(key)}/) do
+      # to_json puts the value in quotes, so drop them: a"b => a\"b
+      $1 ? $1.gsub(key) { value.to_json[1..-2] } : value
     end
-    JSON.parse(file_contents)
   end
 
   def template_exists?(name, host = nil)

data/lib/fluent/plugin/out_opensearch.rb

@@ -95,13 +95,26 @@ module Fluent::Plugin
     DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
 
+    # A placeholder in `host`/`hosts` is expanded from the tag or a record
+    # field, so a log sender chooses its value. Without a check the value
+    # could add a host with ",", userinfo with "@" or a path with "/", and
+    # the configured user/password and custom_headers would go to that host.
+    HOST_PLACEHOLDER_PATTERN = /\$\{[^}]*\}/
+    HOST_LABEL_PATTERN = /\A[0-9A-Za-z_\-]+\z/
+    HOST_NAME_PATTERN = /\A[0-9A-Za-z_\-]+(?:\.[0-9A-Za-z_\-]+)*\z/
+    FIXED_SUFFIX_PATTERN = /\A[0-9A-Za-z_\-]*(?:\.[0-9A-Za-z_\-]+){2,}\.?\z/
+    # A host name or an IPv6 address in "[]", and the port that may follow it.
+    HOST_AND_PORT_PATTERN = /\A(\[[^\]]+\]|[^:]*)(?::([0-9]+))?\z/
+
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
     config_param :user, :string, :default => nil
     config_param :password, :string, :default => nil, :secret => true
     config_param :path, :string, :default => nil
     config_param :scheme, :enum, :list => [:https, :http], :default => :http
-    config_param :hosts, :string, :default => nil
+    # `hosts` accepts "https://user:password@host" and "%{user}:%{password}@",
+    # so it is masked in the configuration dump like `password` is.
+    config_param :hosts, :string, :default => nil, :secret => true
     config_param :target_index_key, :string, :default => nil
     config_param :time_key_format, :string, :default => nil
     config_param :time_precision, :integer, :default => 9
@@ -185,7 +198,8 @@ module Fluent::Plugin
 
     config_section :endpoint, multi: false do
       config_param :region, :string
-      config_param :url do |c|
+      # The URL can carry a user and a password, so mask it in the dump.
+      config_param :url, secret: true do |c|
         c.chomp("/")
       end
       config_param :access_key_id, :string, :default => ""
@@ -211,6 +225,8 @@ module Fluent::Plugin
 
     def initialize
       super
+      # `configure` already builds clients, so this cannot wait until then.
+      @client_mutex = Mutex.new
     end
 
     ######################################################################################################
@@ -225,13 +241,13 @@ module Fluent::Plugin
         if conf[:assume_role_arn].nil?
           aws_container_credentials_relative_uri = conf[:ecs_container_credentials_relative_uri] || ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
           if aws_container_credentials_relative_uri.nil?
-            credentials = Aws::SharedCredentials.new({retries: 2}).credentials rescue nil
-            credentials ||= Aws::InstanceProfileCredentials.new.credentials rescue nil
-            credentials ||= Aws::ECSCredentials.new.credentials
+            credentials = usable_credentials_provider { Aws::SharedCredentials.new }
+            credentials ||= usable_credentials_provider { Aws::InstanceProfileCredentials.new }
+            credentials ||= usable_credentials_provider { Aws::ECSCredentials.new }
           else
             credentials = Aws::ECSCredentials.new({
                             credential_path: aws_container_credentials_relative_uri
-                          }).credentials
+                          })
           end
         else
           if conf[:assume_role_web_identity_token_file].nil?
@@ -239,21 +255,28 @@ module Fluent::Plugin
                             role_arn: conf[:assume_role_arn],
                             role_session_name: conf[:assume_role_session_name],
                             region: sts_creds_region(conf)
-                          }).credentials
+                          })
           else
             credentials = Aws::AssumeRoleWebIdentityCredentials.new({
                             role_arn: conf[:assume_role_arn],
                             web_identity_token_file: conf[:assume_role_web_identity_token_file],
                             region: sts_creds_region(conf)
-                          }).credentials
+                          })
           end
         end
       end
-      raise "No valid AWS credentials found." unless credentials.set?
+      raise "No valid AWS credentials found." unless credentials&.credentials&.set?
 
       credentials
     end
 
+    def usable_credentials_provider
+      provider = yield
+      provider if provider.credentials&.set?
+    rescue
+      nil
+    end
+
     def sts_creds_region(conf)
       conf[:sts_credentials_region] || conf[:region]
     end
@@ -309,6 +332,8 @@ module Fluent::Plugin
         log.info "host placeholder and template installation makes your OpenSearch cluster a bit slow down(beta)."
       end
 
+      warn_host_placeholder_with_credentials
+
       @template_names = []
       if !dry_run?
         if @template_name && @template_file
@@ -339,8 +364,7 @@ module Fluent::Plugin
           }
         end
       end
-      # If AWS credentials is set, consider to expire credentials information forcibly before expired.
-      @credential_mutex = Mutex.new
+      # Credential providers refresh themselves near expiration. This timer recreates them forcibly.
       if @endpoint
         @_aws_credentials = aws_credentials(@endpoint)
 
@@ -348,12 +372,14 @@ module Fluent::Plugin
           timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
             log.debug('Recreate the AWS credentials')
 
-            @credential_mutex.synchronize do
-              @_os = nil
-              begin
-                @_aws_credentials = aws_credentials(@endpoint)
-              rescue => e
-                log.error("Failed to get new AWS credentials: #{e}")
+            begin
+              credentials = aws_credentials(@endpoint)
+            rescue => e
+              log.error("Failed to get new AWS credentials: #{e}")
+            else
+              @client_mutex.synchronize do
+                @_os = nil
+                @_aws_credentials = credentials
               end
             end
           end
@@ -369,7 +395,7 @@ module Fluent::Plugin
           OpenSearch::API.settings[:serializer] = Fluent::Plugin::Serializer::Oj
         end
       rescue LoadError
-        @dump_proc = Yajl.method(:dump)
+        @dump_proc = JSON.method(:generate)
       end
 
       raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
@@ -474,6 +500,34 @@ module Fluent::Plugin
       end
     end
 
+    # A placeholder decides the host, so every credential the configuration
+    # holds follows it. `endpoint` is left alone because
+    # `get_connection_options` ignores the expanded host there.
+    def warn_host_placeholder_with_credentials
+      return if @endpoint
+
+      elements = (@hosts || @host).split(',')
+      return unless elements.any? { |element| element.match?(HOST_PLACEHOLDER_PATTERN) }
+
+      param_name = @hosts ? 'hosts' : 'host'
+      if @user || @password || !@custom_headers.empty?
+        log.warn "'#{param_name}' uses a placeholder, so 'user', 'password' and 'custom_headers' are sent to whichever host a tag or a record field expands to. Make sure that only trusted senders can set them."
+      end
+      if elements.any? { |element| credentials_before_placeholder?(element) }
+        # Do not put the element in the log: it might hold the password.
+        log.warn "'#{param_name}' has a user and password in front of a host that a placeholder decides, so they are sent to whichever host a tag or a record field expands to. Make sure that only trusted senders can set it."
+      end
+    end
+
+    # A user and password written in a URL sit in front of the host, so a
+    # placeholder that comes after them picks where they are sent.
+    def credentials_before_placeholder?(element)
+      userinfo = element.index('@')
+      placeholder = element.index('${')
+
+      !userinfo.nil? && !placeholder.nil? && userinfo < placeholder
+    end
+
     def placeholder?(name, param)
       placeholder_validities = []
       placeholder_validators(name, param).each do |v|
@@ -615,6 +669,14 @@ module Fluent::Plugin
     end
 
     def client(host = nil, compress_connection = false)
+      # Flush threads share @_os, and each of them may ask for a different
+      # host. Without this lock a thread can pass the check below and then
+      # pick up the client another thread just built for its own host, which
+      # sends the chunk to the wrong cluster.
+      @client_mutex.synchronize { build_or_reuse_client(host, compress_connection) }
+    end
+
+    def build_or_reuse_client(host, compress_connection)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
@@ -630,7 +692,7 @@ module Fluent::Plugin
                              :aws_sigv4,
                              service: @endpoint.aws_service_name.to_s,
                              region: @endpoint.region,
-                             credentials: @_aws_credentials,
+                             credentials_provider: @_aws_credentials,
                            )
 
                            f.adapter @http_backend, @backend_options
@@ -840,6 +902,63 @@ module Fluent::Plugin
       return logstash_prefix, logstash_dateformat, index_name, template_name, customize_template, application_name, pipeline
     end
 
+    def expand_host_placeholders(chunk)
+      template = @hosts || @host
+      template.split(',').each do |element|
+        offset = 0
+        while (match = HOST_PLACEHOLDER_PATTERN.match(element, offset))
+          offset = match.end(0)
+          value = extract_placeholders(match[0], chunk)
+          next if valid_host_placeholder_value?(value, match.pre_match, match.post_match)
+
+          raise UnrecoverableRequestFailure, "Rejected #{value.dump} for #{match[0]} in 'host'/'hosts': a placeholder may only fill in the part of a host name that it stands for."
+        end
+      end
+
+      extract_placeholders(template, chunk)
+    end
+
+    # What a value may hold depends on where the operator put the placeholder.
+    # `before` and `after` are the parts of the element around it.
+    def valid_host_placeholder_value?(value, before, after)
+      # The operator wrote the "[]" that an IPv6 address needs.
+      return Resolv::IPv6::Regex.match?(value) if before.end_with?('[') && after.start_with?(']')
+
+      unless whole_host_placeholder?(before, after)
+        # The value continues a name the operator wrote, so a port cannot
+        # follow it and a "." may only add a sub domain to a fixed domain.
+        pattern = fixed_domain_suffix?(after) ? HOST_NAME_PATTERN : HOST_LABEL_PATTERN
+        return pattern.match?(value)
+      end
+
+      host, port = value.match(HOST_AND_PORT_PATTERN)&.captures
+      # The value may end with a port, unless the operator wrote one already.
+      return false if host.nil? || (port && after.start_with?(':'))
+
+      HOST_NAME_PATTERN.match?(host) || bracketed_ipv6?(host, before)
+    end
+
+    # The value fills in the whole host name when only a scheme or a userinfo
+    # comes before it and the host name ends where the value ends.
+    def whole_host_placeholder?(before, after)
+      (before.empty? || before.end_with?('//', '@')) &&
+        (after.empty? || after.start_with?(':', '/', '?', '#'))
+    end
+
+    # The domain is fixed when two or more names follow the last placeholder,
+    # because the last labels of the host name then come from the operator.
+    # One name, such as ".com", lets a value with a "." pick any name under it.
+    def fixed_domain_suffix?(after)
+      host_part = after[/\A[^:\/?#]*/]
+      FIXED_SUFFIX_PATTERN.match?(host_part.split(HOST_PLACEHOLDER_PATTERN, -1).last.to_s)
+    end
+
+    # `URI()` in `get_connection_options` needs a scheme to read an IPv6
+    # address, and the operator writes the scheme, not the value.
+    def bracketed_ipv6?(host, before)
+      !before.empty? && host.start_with?('[') && Resolv::IPv6::Regex.match?(host[1..-2])
+    end
+
     def multi_workers_ready?
       true
     end
@@ -862,11 +981,7 @@ module Fluent::Plugin
       tag = chunk.metadata.tag
       chunk_id = dump_unique_id_hex(chunk.unique_id)
       extracted_values = expand_placeholders(chunk)
-      host = if @hosts
-               extract_placeholders(@hosts, chunk)
-             else
-               extract_placeholders(@host, chunk)
-             end
+      host = expand_host_placeholders(chunk)
 
       affinity_target_indices = get_affinity_target_indices(chunk)
       chunk.msgpack_each do |time, record|
@@ -940,7 +1055,7 @@ module Fluent::Plugin
             {"_index" => {"order" => "desc"}}
          ]
         }
-        result = client.search(options.merge(:body => Yajl.dump(query)))
+        result = client.search(options.merge(:body => JSON.generate(query)))
         # There should be just one hit per _id, but in case there still is multiple, just the oldest index is stored to map
         result['hits']['hits'].each do |hit|
           indices[hit["_id"]] = hit["_index"]
@@ -1130,8 +1245,14 @@ module Fluent::Plugin
 
         log.warn "Exception ignored in tag #{tag}: #{e.class.name} #{e.message}" if ignore
 
-        @_os = nil if @reconnect_on_error
-        @_os_info = nil if @reconnect_on_error
+        if @reconnect_on_error
+          # Another flush thread may be building a client right now, so drop the
+          # cached one under the lock that guards it.
+          @client_mutex.synchronize do
+            @_os = nil
+            @_os_info = nil
+          end
+        end
 
         raise UnrecoverableRequestFailure if ignore && @exception_backup
 

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -155,11 +155,7 @@ module Fluent::Plugin
       data_stream_template_name = @data_stream_template_name
       host = nil
       if @use_placeholder
-        host = if @hosts
-                 extract_placeholders(@hosts, chunk)
-               else
-                 extract_placeholders(@host, chunk)
-               end
+        host = expand_host_placeholders(chunk)
         data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
         begin

data/test/plugin/test_in_opensearch.rb

@@ -322,6 +322,21 @@ class OpenSearchInputTest < Test::Unit::TestCase
     assert_equal 'raw.opensearch', instance.tag
   end
 
+  def test_hosts_is_masked_in_the_configuration_dump
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+        'hosts' => 'https://john:passw0rd@host1:443/elastic/',
+        'tag' => 'raw.opensearch',
+        'check_connection' => 'false',
+      }, [])
+    driver(config)
+
+    dump = config.to_masked_element.to_s
+    assert_false dump.include?('passw0rd')
+    assert_true dump.include?('hosts xxxxxx')
+  end
+
   def test_hosts_list_with_escape_placeholders
     config = %{
       hosts    https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2

data/test/plugin/test_opensearch_error_handler.rb

@@ -596,7 +596,6 @@ class TestOpenSearchErrorHandler < Test::Unit::TestCase
             "_type"  : "bar",
             "_id" : "6",
             "status" : 500,
-            "_type"  : "bar",
             "error" : {
               "type" : "out_of_memory_error",
               "reason":"Java heap space"

data/test/plugin/test_out_opensearch.rb

@@ -85,6 +85,53 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     }
   end
 
+  def chained_credentials_conf
+    {
+      access_key_id: '',
+      secret_access_key: '',
+      assume_role_arn: nil,
+      ecs_container_credentials_relative_uri: nil,
+    }
+  end
+
+  def with_stubbed_aws_responses
+    saved = Aws.config.key?(:stub_responses) ? Aws.config[:stub_responses] : :__unset__
+    Aws.config[:stub_responses] = true
+    yield
+  ensure
+    if saved == :__unset__
+      Aws.config.delete(:stub_responses)
+    else
+      Aws.config[:stub_responses] = saved
+    end
+  end
+
+  def driver_with_captured_credential_refresh
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+      }, [
+        Fluent::Config::Element.new('endpoint', '', {
+                                      'url' => "https://search-opensearch.aws.example.com/",
+                                      'region' => "local",
+                                      'access_key_id' => 'YOUR_AWESOME_KEY',
+                                      'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                      'refresh_credentials_interval' => '18000',
+                                    }, []),
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+      ])
+    d = Fluent::Test::Driver::Output.new(Fluent::Plugin::OpenSearchOutput)
+    timers = {}
+    d.instance.define_singleton_method(:timer_execute) do |title, interval, &block|
+      timers[title] = block
+    end
+    # Keep `configure` off the network.
+    d.instance.define_singleton_method(:detect_os_major_version) { 1 }
+    d.configure(config)
+
+    [d, timers[:out_opensearch_expire_credentials]]
+  end
+
   def stub_opensearch_info(url="http://localhost:9200/", version="1.2.2")
     body ="{\"version\":{\"number\":\"#{version}\", \"distribution\":\"opensearch\"},\"tagline\":\"The OpenSearch Project: https://opensearch.org/\"}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
@@ -315,9 +362,141 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     assert_equal "fluentd", instance.endpoint.assume_role_session_name
     assert_nil instance.endpoint.assume_role_web_identity_token_file
     assert_nil instance.endpoint.sts_credentials_region
+    assert_equal 18000, instance.endpoint.refresh_credentials_interval
     assert_equal :es, instance.endpoint.aws_service_name
   end
 
+  test 'hosts is masked in the configuration dump' do
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+        'hosts' => 'https://john:passw0rd@host1:443/elastic/',
+      }, [
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+      ])
+    driver(config)
+
+    dump = config.to_masked_element.to_s
+    assert_false dump.include?('passw0rd')
+    assert_true dump.include?('hosts xxxxxx')
+  end
+
+  test 'endpoint url is masked in the configuration dump' do
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+      }, [
+        Fluent::Config::Element.new('endpoint', '', {
+                                      'url' => "https://john:passw0rd@search-opensearch.aws.example.com/",
+                                      'region' => "local",
+                                      'access_key_id' => 'YOUR_AWESOME_KEY',
+                                      'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                    }, []),
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+      ])
+    driver(config)
+
+    dump = config.to_masked_element.to_s
+    assert_false dump.include?('passw0rd')
+    assert_true dump.include?('url xxxxxx')
+  end
+
+  test 'aws_credentials returns credential provider with access key' do
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+      }, [
+        Fluent::Config::Element.new('endpoint', '', {
+                                      'url' => "https://search-opensearch.aws.example.com/",
+                                      'region' => "local",
+                                      'access_key_id' => 'YOUR_AWESOME_KEY',
+                                      'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                    }, []),
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+
+      ])
+    instance = driver(config).instance
+    credentials = instance.aws_credentials(instance.endpoint)
+
+    assert_instance_of Aws::Credentials, credentials
+    assert_equal 'YOUR_AWESOME_KEY', credentials.credentials.access_key_id
+  end
+
+  test 'aws_credentials returns refreshable credential provider with assume_role_arn' do
+    with_stubbed_aws_responses do
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'opensearch',
+        }, [
+          Fluent::Config::Element.new('endpoint', '', {
+                                        'url' => "https://search-opensearch.aws.example.com/",
+                                        'region' => "local",
+                                        'assume_role_arn' => 'arn:aws:iam::123456789012:role/some-role',
+                                      }, []),
+          Fluent::Config::Element.new('buffer', 'tag', {}, [])
+
+        ])
+      instance = driver(config).instance
+      credentials = instance.aws_credentials(instance.endpoint)
+
+      assert_instance_of Aws::AssumeRoleCredentials, credentials
+      assert_kind_of Aws::RefreshingCredentials, credentials
+      assert_true credentials.credentials.set?
+    end
+  end
+
+  test 'aws_credentials falls back to the next provider when a provider has no credentials' do
+    unset_provider = flexmock('unset provider', credentials: Aws::Credentials.new(nil, nil))
+    ecs_provider = flexmock('ecs provider', credentials: Aws::Credentials.new('ECS_KEY', 'ECS_SECRET'))
+    flexmock(Aws::SharedCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::InstanceProfileCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::ECSCredentials).should_receive(:new).and_return(ecs_provider)
+    instance = driver.instance
+
+    assert_same ecs_provider, instance.aws_credentials(chained_credentials_conf)
+  end
+
+  test 'aws_credentials raises when no provider has usable credentials' do
+    unset_provider = flexmock('unset provider', credentials: Aws::Credentials.new(nil, nil))
+    flexmock(Aws::SharedCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::InstanceProfileCredentials).should_receive(:new).and_return(unset_provider)
+    flexmock(Aws::ECSCredentials).should_receive(:new)
+      .and_raise(ArgumentError.new("Cannot instantiate an ECS Credential Provider without a credential path or endpoint."))
+    instance = driver.instance
+
+    assert_raise(RuntimeError.new("No valid AWS credentials found.")) do
+      instance.aws_credentials(chained_credentials_conf)
+    end
+  end
+
+  test 'credentials provider is passed to aws_sigv4 middleware' do
+    with_stubbed_aws_responses do
+      config = Fluent::Config::Element.new(
+        'ROOT', '', {
+          '@type' => 'opensearch',
+        }, [
+          Fluent::Config::Element.new('endpoint', '', {
+                                        'url' => "https://search-opensearch.aws.example.com/",
+                                        'region' => "local",
+                                        'assume_role_arn' => 'arn:aws:iam::123456789012:role/some-role',
+                                      }, []),
+          Fluent::Config::Element.new('buffer', 'tag', {}, [])
+
+        ])
+      instance = driver(config).instance
+      # Reaching into Faraday internals. If this breaks after a Faraday or
+      # faraday_middleware-aws-sigv4 update, check RackBuilder::Handler.
+      handler = instance.client.transport.transport.connections.first.connection.builder.handlers.find { |h|
+        h.klass == FaradayMiddleware::AwsSigV4
+      }
+
+      assert_not_nil handler
+      signer = handler.build(proc {}).instance_variable_get(:@signer)
+      assert_instance_of Aws::AssumeRoleCredentials, signer.credentials_provider
+      assert_same instance.instance_variable_get(:@_aws_credentials), signer.credentials_provider
+    end
+  end
+
   data("OpenSearch Service" => [:es, 'es'],
        "OpenSearch Serverless" => [:aoss, 'aoss'])
   test 'configure endpoint section w/ aws_service_name' do |data|
@@ -989,6 +1168,53 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     assert_requested(:put, "https://logs.google.com:777/os//#{endpoint}/myapp_alias_template-test.template", times: 1)
   end
 
+  # A customize_template value can come from a tag or a record field.
+  INJECTED_TEMPLATE_VALUE = 'x": {}}, "index_patterns": ["*"], "order": 2147483647, "aliases": {"y'
+
+  def custom_template(template_file_name, appid)
+    template_file = File.join(File.dirname(__FILE__), template_file_name)
+    driver.instance.get_custom_template(template_file,
+                                        {"--appid--" => appid, "--index_prefix--" => "mylogs"})
+  end
+
+  data("legacy_template" => ['test_alias_template.json',
+                             %w(aliases mappings order settings template)],
+       "new_template"    => ['test_index_alias_template.json',
+                             %w(index_patterns priority template)])
+  def test_custom_template_value_cannot_add_template_keys(data)
+    template_file_name, expected_keys = data
+
+    template = custom_template(template_file_name, INJECTED_TEMPLATE_VALUE)
+
+    assert_equal(expected_keys, template.keys.sort)
+  end
+
+  def test_custom_template_value_stays_a_single_string
+    template = custom_template('test_alias_template.json', INJECTED_TEMPLATE_VALUE)
+
+    assert_equal(5, template["order"])
+    assert_equal(["#{INJECTED_TEMPLATE_VALUE}-alias"], template["aliases"].keys)
+  end
+
+  def test_custom_template_value_keeps_a_backslash
+    template = custom_template('test_alias_template.json', 'a\0b')
+
+    assert_equal(['a\0b-alias'], template["aliases"].keys)
+  end
+
+  # A key is not always inside a string. If JSON is expected there, the value
+  # is that piece of JSON.
+  def test_custom_template_value_outside_a_string_stays_json
+    template_file = File.join(File.dirname(__FILE__), 'test_settings_template.json')
+
+    template = driver.instance.get_custom_template(template_file,
+                                                   {"--settings--" => '{"number_of_shards": 5}',
+                                                    "--index_prefix--" => "mylogs"})
+
+    assert_equal({"number_of_shards" => 5}, template["settings"])
+    assert_equal("mylogs*", template["template"])
+  end
+
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])
   def test_custom_template_installation_for_host_placeholder(data)
@@ -1774,6 +2000,43 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  # Flush threads share @_os, so only one of them may look at it at a time.
+  def test_client_does_not_let_flush_threads_overlap
+    instance = driver.instance
+    counter = Mutex.new
+    inside = 0
+    max_inside = 0
+
+    instance.define_singleton_method(:get_connection_options) do |con_host = nil|
+      counter.synchronize do
+        inside += 1
+        max_inside = inside if inside > max_inside
+      end
+      sleep 0.05
+      counter.synchronize { inside -= 1 }
+      {hosts: [{host: con_host, port: 9200, scheme: 'http'}]}
+    end
+
+    threads = 3.times.map { |i| Thread.new { instance.client("host-#{i}") } }
+    threads.each(&:join)
+
+    assert_equal(1, max_inside)
+  end
+
+  def test_client_returns_a_client_for_the_host_it_was_asked_for
+    instance = driver.instance
+
+    results = 3.times.map { |i|
+      Thread.new do
+        5.times.map { instance.client("host-#{i}:9201").transport.transport.hosts.first[:host] }
+      end
+    }.map(&:value)
+
+    results.each_with_index do |hosts, i|
+      assert_equal(["host-#{i}"], hosts.uniq)
+    end
+  end
+
   def test_password_is_required_if_specify_user
     config = %{
       user john
@@ -2758,6 +3021,149 @@ class OpenSearchOutputTest < Test::Unit::TestCase
       connection_spec = {host: "myhost-1", port: 9200, scheme: "http"}
       assert_equal("could not push logs to OpenSearch cluster (#{connection_spec.inspect}): [503] ", exception.message)
     end
+
+    # `extract_placeholders` needs a real chunk, so replace it with the values
+    # that a crafted tag or record field would expand to. A bare value is the
+    # one for "${pipeline_id}".
+    def expand_hosts_with(hosts, placeholder_values)
+      values = placeholder_values.is_a?(Hash) ? placeholder_values : {"pipeline_id" => placeholder_values}
+      instance = driver.configure("hosts #{hosts}").instance
+      instance.define_singleton_method(:extract_placeholders) do |value, _chunk|
+        values.inject(value) { |expanded, (name, replacement)| expanded.gsub("${#{name}}", replacement) }
+      end
+      instance.expand_host_placeholders(nil)
+    end
+
+    data("host name"   => ["${pipeline_id},logs2.example.com:9201", "os.internal"],
+         "with port"   => ["${pipeline_id},logs2.example.com:9201", "os.internal:9200"],
+         "in a url"    => ["https://${pipeline_id}/os", "logs.example.com:9201"],
+         "ipv6"        => ["http://${pipeline_id}:9201", "[2001:db8::1]"],
+         "ipv6 + port" => ["http://${pipeline_id}/os", "[2001:db8::1]:9201"],
+         "ipv6 in []"  => ["http://[${pipeline_id}]:9201", "2001:db8::1"])
+    def test_accepts_placeholder_value_which_fills_in_the_whole_host(data)
+      hosts, placeholder_value = data
+      assert_equal(hosts.sub('${pipeline_id}', placeholder_value),
+                   expand_hosts_with(hosts, placeholder_value))
+    end
+
+    data("extra host"   => "evil.example.com,myhost",
+         "userinfo"     => "1@evil.example.com",
+         "path"         => "1/evil.example.com",
+         "space"        => "1 evil.example.com",
+         "other domain" => "evil.example.net",
+         "port"         => "1:9999",
+         "empty"        => "")
+    def test_rejects_placeholder_value_which_does_not_fit_its_place(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("myhost-${pipeline_id},logs2.example.com:9201", placeholder_value)
+      }
+    end
+
+    # The operator wrote ".example.com", so a tag such as "nginx.access" only
+    # adds a sub domain to it.
+    def test_accepts_placeholder_value_with_a_sub_domain
+      assert_equal("https://logs-nginx.access.example.com/os",
+                   expand_hosts_with("https://logs-${pipeline_id}.example.com/os", "nginx.access"))
+    end
+
+    # ".example.com" is fixed even though another placeholder stands before it.
+    def test_accepts_placeholder_value_with_a_sub_domain_before_another_placeholder
+      assert_equal("https://nginx.access.us.example.com/os",
+                   expand_hosts_with("https://${pipeline_id}.${region}.example.com/os",
+                                     {"pipeline_id" => "nginx.access", "region" => "us"}))
+    end
+
+    # A trailing "." or a single name after the placeholder fixes no domain.
+    data("trailing dot" => ["myhost-${pipeline_id}.", "evil.example.com"],
+         "one name"     => ["https://logs-${pipeline_id}.com/os", "evil.attacker"])
+    def test_rejects_placeholder_value_which_leaves_the_configured_domain(data)
+      hosts, placeholder_value = data
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with(hosts, placeholder_value)
+      }
+    end
+
+    data("extra host" => "evil.example.com,myhost",
+         "port"       => "1:9999",
+         "empty"      => "")
+    def test_rejects_placeholder_value_which_is_not_a_sub_domain(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("https://logs-${pipeline_id}.example.com/os", placeholder_value)
+      }
+    end
+
+    # `get_connection_options` reads an element with `URI()`, which cannot
+    # read an IPv6 address when the element has no scheme.
+    data("empty"     => "",
+         "port only" => ":9999",
+         "two ports" => "os.internal:9200:9201",
+         "path"      => "myhost/evil.example.com",
+         "ipv6"      => "[2001:db8::1]")
+    def test_rejects_placeholder_value_which_is_not_a_host_name(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("${pipeline_id},logs2.example.com:9201", placeholder_value)
+      }
+    end
+
+    data("host name" => "os.internal:9200",
+         "ipv6"      => "[2001:db8::1]:9200")
+    def test_rejects_placeholder_value_with_a_port_when_the_port_is_configured(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("http://${pipeline_id}:9201", placeholder_value)
+      }
+    end
+
+    data("in []"      => ["http://[${pipeline_id}]:9201", "2001:db8"],
+         "extra host" => ["http://[${pipeline_id}]:9201", "[2001:db8::1],[2001:db8::2]"],
+         "whole host" => ["http://${pipeline_id}:9201", "[2001:db8]"])
+    def test_rejects_placeholder_value_which_is_not_an_ipv6_address(data)
+      hosts, placeholder_value = data
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with(hosts, placeholder_value)
+      }
+    end
+
+    # The operator writes the structure, so only the placeholder value is
+    # checked. Every form that `get_connection_options` accepts keeps working.
+    data("plain host" => ["myhost-${pipeline_id},logs2.example.com:9201",
+                          "myhost-1,logs2.example.com:9201"],
+         "url"        => ["https://logs-${pipeline_id}.example.com:9201/os",
+                          "https://logs-1.example.com:9201/os"],
+         "userinfo"   => ["https://john:pass@logs-${pipeline_id}.example.com/os",
+                          "https://john:pass@logs-1.example.com/os"],
+         "ipv6"       => ["http://[2001:db8::1]:9201,myhost-${pipeline_id}",
+                          "http://[2001:db8::1]:9201,myhost-1"])
+    def test_keeps_the_configured_host_structure(data)
+      hosts, expected = data
+      assert_equal(expected, expand_hosts_with(hosts, "1"))
+    end
+
+    def test_warns_when_placeholder_host_is_used_with_credentials
+      d = driver(%{
+        host     logs-${tag}.example.com
+        user     john
+        password doe
+        @log_level info
+      })
+      assert_true(d.logs.any? { |log| log.include?("'host' uses a placeholder") })
+    end
+
+    def test_warns_when_hosts_embeds_credentials_in_front_of_a_placeholder
+      d = driver(%{
+        hosts https://john:secret@logs-${tag}.example.com/os
+        @log_level info
+      })
+      assert_true(d.logs.any? { |log| log.include?("in front of a host that a placeholder decides") })
+      assert_false(d.logs.any? { |log| log.include?("secret") })
+    end
+
+    def test_does_not_warn_when_credentials_belong_to_a_static_host
+      d = driver(%{
+        hosts https://john:secret@static.example.com/os,logs-${tag}.example.com
+        @log_level info
+      })
+      assert_false(d.logs.any? { |log| log.include?("in front of a host that a placeholder decides") })
+    end
   end
 
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
@@ -3648,7 +4054,6 @@ class OpenSearchOutputTest < Test::Unit::TestCase
                 "_index" : "foo",
                 "_type"  : "bar",
                 "_id" : "abc",
-                "_id" : "abc",
                 "status" : 409
               }
             }
@@ -4000,4 +4405,43 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     d.run
     assert { d.logs.any?(/\[error\]: Failed to get new AWS credentials: No valid AWS credentials found.\n/) }
   end
+
+  test 'a failed credential refresh keeps the current credentials and client' do
+    d, refresh = driver_with_captured_credential_refresh
+    instance = d.instance
+    credentials = instance.instance_variable_get(:@_aws_credentials)
+    instance.instance_variable_set(:@_os, :cached_client)
+    flexmock(instance).should_receive(:aws_credentials)
+      .and_raise(::RuntimeError.new("No valid AWS credentials found."))
+
+    refresh.call
+
+    assert_same credentials, instance.instance_variable_get(:@_aws_credentials)
+    assert_equal :cached_client, instance.instance_variable_get(:@_os)
+    assert { d.logs.any?(/Failed to get new AWS credentials: No valid AWS credentials found./) }
+  end
+
+  # The refresh shares @_os with the flush threads, so it has to use their lock
+  # instead of one of its own.
+  test 'a credential refresh swaps the credentials under the client lock' do
+    d, refresh = driver_with_captured_credential_refresh
+    instance = d.instance
+    new_credentials = Aws::Credentials.new('NEW_KEY', 'NEW_SECRET')
+    flexmock(instance).should_receive(:aws_credentials).and_return(new_credentials)
+    instance.instance_variable_set(:@_os, :cached_client)
+
+    # Records what the plugin had done by the time it left the lock.
+    seen = []
+    fake_lock = Object.new
+    fake_lock.define_singleton_method(:synchronize) do |&block|
+      block.call
+      seen << [instance.instance_variable_get(:@_aws_credentials),
+               instance.instance_variable_get(:@_os)]
+    end
+    instance.instance_variable_set(:@client_mutex, fake_lock)
+
+    refresh.call
+
+    assert_equal [[new_credentials, nil]], seen
+  end
 end

data/test/plugin/test_settings_template.json

@@ -0,0 +1,4 @@
+{
+  "template": "--index_prefix--*",
+  "settings": --settings--
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.8
 platform: ruby
 authors:
 - Hiroshi Hatake
@@ -249,6 +249,7 @@ files:
 - test/plugin/test_opensearch_tls.rb
 - test/plugin/test_out_opensearch.rb
 - test/plugin/test_out_opensearch_data_stream.rb
+- test/plugin/test_settings_template.json
 - test/plugin/test_template.json
 - test/test_log-ext.rb
 homepage: https://github.com/fluent/fluent-plugin-opensearch
@@ -270,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector
 test_files:
@@ -287,5 +288,6 @@ test_files:
 - test/plugin/test_opensearch_tls.rb
 - test/plugin/test_out_opensearch.rb
 - test/plugin/test_out_opensearch_data_stream.rb
+- test/plugin/test_settings_template.json
 - test/plugin/test_template.json
 - test/test_log-ext.rb