fluent-plugin-opensearch 1.1.7 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59ef255b7e5cb5ceae9b97b0678cdc7151f0514263934f21e1028a45220180d6
-  data.tar.gz: 7603936854d8a36c74b6fa6ee4187b089cd781c997da8b97eed21fafbd52833c
+  metadata.gz: '04843ec3381adb39f0e7fbc2a6cc76c25fa8d1d85d5733e500e58d7814beb57a'
+  data.tar.gz: b5e60e70521053e723b9c4df9953b03774a1207f7cf3c19b9c3aeb385e03f15f
 SHA512:
-  metadata.gz: 4caf0aa8ff2803a46536db82defff7b203f837dea3e2fc97b46487a04c9d85c5b4f71b9145985e9616e4db54f4bdef93a61ca85d941612474794276bdbf37531
-  data.tar.gz: b9fb24c499f3c84ece032e744a567ee9c34b7b8a2f47c39a706408e80fb7c8f19624499d447ed4191e75333c4050ef79237e51037dd0122ea95fc7aa6ad6f097
+  metadata.gz: 80cb6808fef85cc16fb835324f2ff3fbe3d42978d3c3f72dccf35251d39b3fe29f74d1271c30b2913959fd7cdc1f4e277ed91608663756e24d3d2e9b438e2943
+  data.tar.gz: 6d936e5608366a827cdd03f8bf51615b34780f9fdb7354a764cb2f39e4c570ad5f768ccc73cf286c9f22a3bafc529868b7e4434c8ea6b001af3328c541a7cb3e

data/.github/workflows/coverage.yaml

@@ -8,7 +8,7 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Ruby 3.0
-        uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+        uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
         with:
           ruby-version: '3.1'
       - name: Build and test with Rake

data/.github/workflows/linux.yml

@@ -26,7 +26,7 @@ jobs:
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/.github/workflows/macos.yml

@@ -26,7 +26,7 @@ jobs:
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/.github/workflows/windows.yml

@@ -26,7 +26,7 @@ jobs:
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 1.1.8
+ - `out_opensearch`: limit what a host placeholder can expand to (#192)
+ - `out_opensearch`: escape a `customize_template` value inside a string (#194)
+ - `out_opensearch`: lock the client cache against flush threads (#195)
+
 ### 1.1.7
  - test: remove duplicate keys from bulk response fixtures (#190)
  - out\_opensearch: keep AWS credential providers refreshable to fix expired SigV4 tokens with assume\_role\_arn (#188)

data/README.md

@@ -1241,6 +1241,12 @@ records = {key1: "value1", key2: "value2"}
 </match>
 ```
 
+#### host and hosts
+
+`host` and `hosts` take placeholders too, for example `hosts https://logs-${tag}.example.com:9201/os`. A value may only fill in the part of a host name that the placeholder stands for, and how much it may fill in depends on the names that you write after the placeholder. `logs-${tag}.example.com` lets a value add a sub domain of `example.com`, such as `nginx.access`. `logs-${tag}.com` lets a value fill in one name only, because a single name fixes the top level domain and nothing else: the name that decides who owns the domain would still come from the value. A value that does not fit raises `UnrecoverableRequestFailure`.
+
+**Note**: the configured `user`, `password` and `custom_headers` are sent to whichever host the value picks. So write a domain that you control, where nobody else can create a sub domain. A shared domain such as `amazonaws.com` is not enough. Without such a domain, let only senders that you trust set the placeholder value.
+
 ## Multi workers
 
 Since Fluentd v0.14, multi workers feature had been implemented to increase throughput with multiple processes. This feature allows Fluentd processes to use one or more CPUs. This feature will be enabled by the following system configuration:

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.1.7'
+  s.version       = '1.1.8'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/in_opensearch.rb

@@ -51,7 +51,9 @@ module Fluent::Plugin
     config_param :password, :string, :default => nil, :secret => true
     config_param :path, :string, :default => nil
     config_param :scheme, :enum, :list => [:https, :http], :default => :http
-    config_param :hosts, :string, :default => nil
+    # `hosts` accepts "https://user:password@host", so it is masked in the
+    # configuration dump like `password` is.
+    config_param :hosts, :string, :default => nil, :secret => true
     config_param :index_name, :string, :default => "fluentd"
     config_param :parse_timestamp, :bool, :default => false
     config_param :timestamp_key_format, :string, :default => nil

data/lib/fluent/plugin/opensearch_index_template.rb

@@ -24,6 +24,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require 'json'
+
 require 'fluent/error'
 require_relative './opensearch_error'
 
@@ -42,11 +44,25 @@ module Fluent::OpenSearchIndexTemplate
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
     customize_template.each do |key, value|
-      file_contents = file_contents.gsub(key,value.downcase)
+      file_contents = substitute_template_value(file_contents, key, value.downcase)
     end
     JSON.parse(file_contents, allow_duplicate_key: true)
   end
 
+  # A value can come from a tag or a record field. If its key is inside a JSON
+  # string ($1), a `"` in the value would end that string, and the rest would
+  # be read as extra template keys, so the value is escaped there. If the key
+  # is anywhere else, the value is a piece of JSON, like a whole settings
+  # object, so it goes in as it is. The block form stops gsub from reading
+  # backslash sequences in the value.
+  def substitute_template_value(file_contents, key, value)
+    json_string = /"(?:\\.|[^"\\])*"/
+    file_contents.gsub(/(#{json_string})|#{Regexp.escape(key)}/) do
+      # to_json puts the value in quotes, so drop them: a"b => a\"b
+      $1 ? $1.gsub(key) { value.to_json[1..-2] } : value
+    end
+  end
+
   def template_exists?(name, host = nil)
     if @use_legacy_template
       client(host).indices.get_template(:name => name)

data/lib/fluent/plugin/out_opensearch.rb

@@ -95,13 +95,26 @@ module Fluent::Plugin
     DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
 
+    # A placeholder in `host`/`hosts` is expanded from the tag or a record
+    # field, so a log sender chooses its value. Without a check the value
+    # could add a host with ",", userinfo with "@" or a path with "/", and
+    # the configured user/password and custom_headers would go to that host.
+    HOST_PLACEHOLDER_PATTERN = /\$\{[^}]*\}/
+    HOST_LABEL_PATTERN = /\A[0-9A-Za-z_\-]+\z/
+    HOST_NAME_PATTERN = /\A[0-9A-Za-z_\-]+(?:\.[0-9A-Za-z_\-]+)*\z/
+    FIXED_SUFFIX_PATTERN = /\A[0-9A-Za-z_\-]*(?:\.[0-9A-Za-z_\-]+){2,}\.?\z/
+    # A host name or an IPv6 address in "[]", and the port that may follow it.
+    HOST_AND_PORT_PATTERN = /\A(\[[^\]]+\]|[^:]*)(?::([0-9]+))?\z/
+
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
     config_param :user, :string, :default => nil
     config_param :password, :string, :default => nil, :secret => true
     config_param :path, :string, :default => nil
     config_param :scheme, :enum, :list => [:https, :http], :default => :http
-    config_param :hosts, :string, :default => nil
+    # `hosts` accepts "https://user:password@host" and "%{user}:%{password}@",
+    # so it is masked in the configuration dump like `password` is.
+    config_param :hosts, :string, :default => nil, :secret => true
     config_param :target_index_key, :string, :default => nil
     config_param :time_key_format, :string, :default => nil
     config_param :time_precision, :integer, :default => 9
@@ -185,7 +198,8 @@ module Fluent::Plugin
 
     config_section :endpoint, multi: false do
       config_param :region, :string
-      config_param :url do |c|
+      # The URL can carry a user and a password, so mask it in the dump.
+      config_param :url, secret: true do |c|
         c.chomp("/")
       end
       config_param :access_key_id, :string, :default => ""
@@ -211,6 +225,8 @@ module Fluent::Plugin
 
     def initialize
       super
+      # `configure` already builds clients, so this cannot wait until then.
+      @client_mutex = Mutex.new
     end
 
     ######################################################################################################
@@ -316,6 +332,8 @@ module Fluent::Plugin
         log.info "host placeholder and template installation makes your OpenSearch cluster a bit slow down(beta)."
       end
 
+      warn_host_placeholder_with_credentials
+
       @template_names = []
       if !dry_run?
         if @template_name && @template_file
@@ -347,7 +365,6 @@ module Fluent::Plugin
         end
       end
       # Credential providers refresh themselves near expiration. This timer recreates them forcibly.
-      @credential_mutex = Mutex.new
       if @endpoint
         @_aws_credentials = aws_credentials(@endpoint)
 
@@ -355,12 +372,14 @@ module Fluent::Plugin
           timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
             log.debug('Recreate the AWS credentials')
 
-            @credential_mutex.synchronize do
-              @_os = nil
-              begin
-                @_aws_credentials = aws_credentials(@endpoint)
-              rescue => e
-                log.error("Failed to get new AWS credentials: #{e}")
+            begin
+              credentials = aws_credentials(@endpoint)
+            rescue => e
+              log.error("Failed to get new AWS credentials: #{e}")
+            else
+              @client_mutex.synchronize do
+                @_os = nil
+                @_aws_credentials = credentials
               end
             end
           end
@@ -481,6 +500,34 @@ module Fluent::Plugin
       end
     end
 
+    # A placeholder decides the host, so every credential the configuration
+    # holds follows it. `endpoint` is left alone because
+    # `get_connection_options` ignores the expanded host there.
+    def warn_host_placeholder_with_credentials
+      return if @endpoint
+
+      elements = (@hosts || @host).split(',')
+      return unless elements.any? { |element| element.match?(HOST_PLACEHOLDER_PATTERN) }
+
+      param_name = @hosts ? 'hosts' : 'host'
+      if @user || @password || !@custom_headers.empty?
+        log.warn "'#{param_name}' uses a placeholder, so 'user', 'password' and 'custom_headers' are sent to whichever host a tag or a record field expands to. Make sure that only trusted senders can set them."
+      end
+      if elements.any? { |element| credentials_before_placeholder?(element) }
+        # Do not put the element in the log: it might hold the password.
+        log.warn "'#{param_name}' has a user and password in front of a host that a placeholder decides, so they are sent to whichever host a tag or a record field expands to. Make sure that only trusted senders can set it."
+      end
+    end
+
+    # A user and password written in a URL sit in front of the host, so a
+    # placeholder that comes after them picks where they are sent.
+    def credentials_before_placeholder?(element)
+      userinfo = element.index('@')
+      placeholder = element.index('${')
+
+      !userinfo.nil? && !placeholder.nil? && userinfo < placeholder
+    end
+
     def placeholder?(name, param)
       placeholder_validities = []
       placeholder_validators(name, param).each do |v|
@@ -622,6 +669,14 @@ module Fluent::Plugin
     end
 
     def client(host = nil, compress_connection = false)
+      # Flush threads share @_os, and each of them may ask for a different
+      # host. Without this lock a thread can pass the check below and then
+      # pick up the client another thread just built for its own host, which
+      # sends the chunk to the wrong cluster.
+      @client_mutex.synchronize { build_or_reuse_client(host, compress_connection) }
+    end
+
+    def build_or_reuse_client(host, compress_connection)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
@@ -847,6 +902,63 @@ module Fluent::Plugin
       return logstash_prefix, logstash_dateformat, index_name, template_name, customize_template, application_name, pipeline
     end
 
+    def expand_host_placeholders(chunk)
+      template = @hosts || @host
+      template.split(',').each do |element|
+        offset = 0
+        while (match = HOST_PLACEHOLDER_PATTERN.match(element, offset))
+          offset = match.end(0)
+          value = extract_placeholders(match[0], chunk)
+          next if valid_host_placeholder_value?(value, match.pre_match, match.post_match)
+
+          raise UnrecoverableRequestFailure, "Rejected #{value.dump} for #{match[0]} in 'host'/'hosts': a placeholder may only fill in the part of a host name that it stands for."
+        end
+      end
+
+      extract_placeholders(template, chunk)
+    end
+
+    # What a value may hold depends on where the operator put the placeholder.
+    # `before` and `after` are the parts of the element around it.
+    def valid_host_placeholder_value?(value, before, after)
+      # The operator wrote the "[]" that an IPv6 address needs.
+      return Resolv::IPv6::Regex.match?(value) if before.end_with?('[') && after.start_with?(']')
+
+      unless whole_host_placeholder?(before, after)
+        # The value continues a name the operator wrote, so a port cannot
+        # follow it and a "." may only add a sub domain to a fixed domain.
+        pattern = fixed_domain_suffix?(after) ? HOST_NAME_PATTERN : HOST_LABEL_PATTERN
+        return pattern.match?(value)
+      end
+
+      host, port = value.match(HOST_AND_PORT_PATTERN)&.captures
+      # The value may end with a port, unless the operator wrote one already.
+      return false if host.nil? || (port && after.start_with?(':'))
+
+      HOST_NAME_PATTERN.match?(host) || bracketed_ipv6?(host, before)
+    end
+
+    # The value fills in the whole host name when only a scheme or a userinfo
+    # comes before it and the host name ends where the value ends.
+    def whole_host_placeholder?(before, after)
+      (before.empty? || before.end_with?('//', '@')) &&
+        (after.empty? || after.start_with?(':', '/', '?', '#'))
+    end
+
+    # The domain is fixed when two or more names follow the last placeholder,
+    # because the last labels of the host name then come from the operator.
+    # One name, such as ".com", lets a value with a "." pick any name under it.
+    def fixed_domain_suffix?(after)
+      host_part = after[/\A[^:\/?#]*/]
+      FIXED_SUFFIX_PATTERN.match?(host_part.split(HOST_PLACEHOLDER_PATTERN, -1).last.to_s)
+    end
+
+    # `URI()` in `get_connection_options` needs a scheme to read an IPv6
+    # address, and the operator writes the scheme, not the value.
+    def bracketed_ipv6?(host, before)
+      !before.empty? && host.start_with?('[') && Resolv::IPv6::Regex.match?(host[1..-2])
+    end
+
     def multi_workers_ready?
       true
     end
@@ -869,11 +981,7 @@ module Fluent::Plugin
       tag = chunk.metadata.tag
       chunk_id = dump_unique_id_hex(chunk.unique_id)
       extracted_values = expand_placeholders(chunk)
-      host = if @hosts
-               extract_placeholders(@hosts, chunk)
-             else
-               extract_placeholders(@host, chunk)
-             end
+      host = expand_host_placeholders(chunk)
 
       affinity_target_indices = get_affinity_target_indices(chunk)
       chunk.msgpack_each do |time, record|
@@ -1137,8 +1245,14 @@ module Fluent::Plugin
 
         log.warn "Exception ignored in tag #{tag}: #{e.class.name} #{e.message}" if ignore
 
-        @_os = nil if @reconnect_on_error
-        @_os_info = nil if @reconnect_on_error
+        if @reconnect_on_error
+          # Another flush thread may be building a client right now, so drop the
+          # cached one under the lock that guards it.
+          @client_mutex.synchronize do
+            @_os = nil
+            @_os_info = nil
+          end
+        end
 
         raise UnrecoverableRequestFailure if ignore && @exception_backup
 

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -155,11 +155,7 @@ module Fluent::Plugin
       data_stream_template_name = @data_stream_template_name
       host = nil
       if @use_placeholder
-        host = if @hosts
-                 extract_placeholders(@hosts, chunk)
-               else
-                 extract_placeholders(@host, chunk)
-               end
+        host = expand_host_placeholders(chunk)
         data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
         begin

data/test/plugin/test_in_opensearch.rb

@@ -322,6 +322,21 @@ class OpenSearchInputTest < Test::Unit::TestCase
     assert_equal 'raw.opensearch', instance.tag
   end
 
+  def test_hosts_is_masked_in_the_configuration_dump
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+        'hosts' => 'https://john:passw0rd@host1:443/elastic/',
+        'tag' => 'raw.opensearch',
+        'check_connection' => 'false',
+      }, [])
+    driver(config)
+
+    dump = config.to_masked_element.to_s
+    assert_false dump.include?('passw0rd')
+    assert_true dump.include?('hosts xxxxxx')
+  end
+
   def test_hosts_list_with_escape_placeholders
     config = %{
       hosts    https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2

data/test/plugin/test_out_opensearch.rb

@@ -106,6 +106,32 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  def driver_with_captured_credential_refresh
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+      }, [
+        Fluent::Config::Element.new('endpoint', '', {
+                                      'url' => "https://search-opensearch.aws.example.com/",
+                                      'region' => "local",
+                                      'access_key_id' => 'YOUR_AWESOME_KEY',
+                                      'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                      'refresh_credentials_interval' => '18000',
+                                    }, []),
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+      ])
+    d = Fluent::Test::Driver::Output.new(Fluent::Plugin::OpenSearchOutput)
+    timers = {}
+    d.instance.define_singleton_method(:timer_execute) do |title, interval, &block|
+      timers[title] = block
+    end
+    # Keep `configure` off the network.
+    d.instance.define_singleton_method(:detect_os_major_version) { 1 }
+    d.configure(config)
+
+    [d, timers[:out_opensearch_expire_credentials]]
+  end
+
   def stub_opensearch_info(url="http://localhost:9200/", version="1.2.2")
     body ="{\"version\":{\"number\":\"#{version}\", \"distribution\":\"opensearch\"},\"tagline\":\"The OpenSearch Project: https://opensearch.org/\"}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
@@ -340,6 +366,41 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     assert_equal :es, instance.endpoint.aws_service_name
   end
 
+  test 'hosts is masked in the configuration dump' do
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+        'hosts' => 'https://john:passw0rd@host1:443/elastic/',
+      }, [
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+      ])
+    driver(config)
+
+    dump = config.to_masked_element.to_s
+    assert_false dump.include?('passw0rd')
+    assert_true dump.include?('hosts xxxxxx')
+  end
+
+  test 'endpoint url is masked in the configuration dump' do
+    config = Fluent::Config::Element.new(
+      'ROOT', '', {
+        '@type' => 'opensearch',
+      }, [
+        Fluent::Config::Element.new('endpoint', '', {
+                                      'url' => "https://john:passw0rd@search-opensearch.aws.example.com/",
+                                      'region' => "local",
+                                      'access_key_id' => 'YOUR_AWESOME_KEY',
+                                      'secret_access_key' => 'YOUR_AWESOME_SECRET',
+                                    }, []),
+        Fluent::Config::Element.new('buffer', 'tag', {}, [])
+      ])
+    driver(config)
+
+    dump = config.to_masked_element.to_s
+    assert_false dump.include?('passw0rd')
+    assert_true dump.include?('url xxxxxx')
+  end
+
   test 'aws_credentials returns credential provider with access key' do
     config = Fluent::Config::Element.new(
       'ROOT', '', {
@@ -1107,6 +1168,53 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     assert_requested(:put, "https://logs.google.com:777/os//#{endpoint}/myapp_alias_template-test.template", times: 1)
   end
 
+  # A customize_template value can come from a tag or a record field.
+  INJECTED_TEMPLATE_VALUE = 'x": {}}, "index_patterns": ["*"], "order": 2147483647, "aliases": {"y'
+
+  def custom_template(template_file_name, appid)
+    template_file = File.join(File.dirname(__FILE__), template_file_name)
+    driver.instance.get_custom_template(template_file,
+                                        {"--appid--" => appid, "--index_prefix--" => "mylogs"})
+  end
+
+  data("legacy_template" => ['test_alias_template.json',
+                             %w(aliases mappings order settings template)],
+       "new_template"    => ['test_index_alias_template.json',
+                             %w(index_patterns priority template)])
+  def test_custom_template_value_cannot_add_template_keys(data)
+    template_file_name, expected_keys = data
+
+    template = custom_template(template_file_name, INJECTED_TEMPLATE_VALUE)
+
+    assert_equal(expected_keys, template.keys.sort)
+  end
+
+  def test_custom_template_value_stays_a_single_string
+    template = custom_template('test_alias_template.json', INJECTED_TEMPLATE_VALUE)
+
+    assert_equal(5, template["order"])
+    assert_equal(["#{INJECTED_TEMPLATE_VALUE}-alias"], template["aliases"].keys)
+  end
+
+  def test_custom_template_value_keeps_a_backslash
+    template = custom_template('test_alias_template.json', 'a\0b')
+
+    assert_equal(['a\0b-alias'], template["aliases"].keys)
+  end
+
+  # A key is not always inside a string. If JSON is expected there, the value
+  # is that piece of JSON.
+  def test_custom_template_value_outside_a_string_stays_json
+    template_file = File.join(File.dirname(__FILE__), 'test_settings_template.json')
+
+    template = driver.instance.get_custom_template(template_file,
+                                                   {"--settings--" => '{"number_of_shards": 5}',
+                                                    "--index_prefix--" => "mylogs"})
+
+    assert_equal({"number_of_shards" => 5}, template["settings"])
+    assert_equal("mylogs*", template["template"])
+  end
+
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])
   def test_custom_template_installation_for_host_placeholder(data)
@@ -1892,6 +2000,43 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  # Flush threads share @_os, so only one of them may look at it at a time.
+  def test_client_does_not_let_flush_threads_overlap
+    instance = driver.instance
+    counter = Mutex.new
+    inside = 0
+    max_inside = 0
+
+    instance.define_singleton_method(:get_connection_options) do |con_host = nil|
+      counter.synchronize do
+        inside += 1
+        max_inside = inside if inside > max_inside
+      end
+      sleep 0.05
+      counter.synchronize { inside -= 1 }
+      {hosts: [{host: con_host, port: 9200, scheme: 'http'}]}
+    end
+
+    threads = 3.times.map { |i| Thread.new { instance.client("host-#{i}") } }
+    threads.each(&:join)
+
+    assert_equal(1, max_inside)
+  end
+
+  def test_client_returns_a_client_for_the_host_it_was_asked_for
+    instance = driver.instance
+
+    results = 3.times.map { |i|
+      Thread.new do
+        5.times.map { instance.client("host-#{i}:9201").transport.transport.hosts.first[:host] }
+      end
+    }.map(&:value)
+
+    results.each_with_index do |hosts, i|
+      assert_equal(["host-#{i}"], hosts.uniq)
+    end
+  end
+
   def test_password_is_required_if_specify_user
     config = %{
       user john
@@ -2876,6 +3021,149 @@ class OpenSearchOutputTest < Test::Unit::TestCase
       connection_spec = {host: "myhost-1", port: 9200, scheme: "http"}
       assert_equal("could not push logs to OpenSearch cluster (#{connection_spec.inspect}): [503] ", exception.message)
     end
+
+    # `extract_placeholders` needs a real chunk, so replace it with the values
+    # that a crafted tag or record field would expand to. A bare value is the
+    # one for "${pipeline_id}".
+    def expand_hosts_with(hosts, placeholder_values)
+      values = placeholder_values.is_a?(Hash) ? placeholder_values : {"pipeline_id" => placeholder_values}
+      instance = driver.configure("hosts #{hosts}").instance
+      instance.define_singleton_method(:extract_placeholders) do |value, _chunk|
+        values.inject(value) { |expanded, (name, replacement)| expanded.gsub("${#{name}}", replacement) }
+      end
+      instance.expand_host_placeholders(nil)
+    end
+
+    data("host name"   => ["${pipeline_id},logs2.example.com:9201", "os.internal"],
+         "with port"   => ["${pipeline_id},logs2.example.com:9201", "os.internal:9200"],
+         "in a url"    => ["https://${pipeline_id}/os", "logs.example.com:9201"],
+         "ipv6"        => ["http://${pipeline_id}:9201", "[2001:db8::1]"],
+         "ipv6 + port" => ["http://${pipeline_id}/os", "[2001:db8::1]:9201"],
+         "ipv6 in []"  => ["http://[${pipeline_id}]:9201", "2001:db8::1"])
+    def test_accepts_placeholder_value_which_fills_in_the_whole_host(data)
+      hosts, placeholder_value = data
+      assert_equal(hosts.sub('${pipeline_id}', placeholder_value),
+                   expand_hosts_with(hosts, placeholder_value))
+    end
+
+    data("extra host"   => "evil.example.com,myhost",
+         "userinfo"     => "1@evil.example.com",
+         "path"         => "1/evil.example.com",
+         "space"        => "1 evil.example.com",
+         "other domain" => "evil.example.net",
+         "port"         => "1:9999",
+         "empty"        => "")
+    def test_rejects_placeholder_value_which_does_not_fit_its_place(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("myhost-${pipeline_id},logs2.example.com:9201", placeholder_value)
+      }
+    end
+
+    # The operator wrote ".example.com", so a tag such as "nginx.access" only
+    # adds a sub domain to it.
+    def test_accepts_placeholder_value_with_a_sub_domain
+      assert_equal("https://logs-nginx.access.example.com/os",
+                   expand_hosts_with("https://logs-${pipeline_id}.example.com/os", "nginx.access"))
+    end
+
+    # ".example.com" is fixed even though another placeholder stands before it.
+    def test_accepts_placeholder_value_with_a_sub_domain_before_another_placeholder
+      assert_equal("https://nginx.access.us.example.com/os",
+                   expand_hosts_with("https://${pipeline_id}.${region}.example.com/os",
+                                     {"pipeline_id" => "nginx.access", "region" => "us"}))
+    end
+
+    # A trailing "." or a single name after the placeholder fixes no domain.
+    data("trailing dot" => ["myhost-${pipeline_id}.", "evil.example.com"],
+         "one name"     => ["https://logs-${pipeline_id}.com/os", "evil.attacker"])
+    def test_rejects_placeholder_value_which_leaves_the_configured_domain(data)
+      hosts, placeholder_value = data
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with(hosts, placeholder_value)
+      }
+    end
+
+    data("extra host" => "evil.example.com,myhost",
+         "port"       => "1:9999",
+         "empty"      => "")
+    def test_rejects_placeholder_value_which_is_not_a_sub_domain(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("https://logs-${pipeline_id}.example.com/os", placeholder_value)
+      }
+    end
+
+    # `get_connection_options` reads an element with `URI()`, which cannot
+    # read an IPv6 address when the element has no scheme.
+    data("empty"     => "",
+         "port only" => ":9999",
+         "two ports" => "os.internal:9200:9201",
+         "path"      => "myhost/evil.example.com",
+         "ipv6"      => "[2001:db8::1]")
+    def test_rejects_placeholder_value_which_is_not_a_host_name(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("${pipeline_id},logs2.example.com:9201", placeholder_value)
+      }
+    end
+
+    data("host name" => "os.internal:9200",
+         "ipv6"      => "[2001:db8::1]:9200")
+    def test_rejects_placeholder_value_with_a_port_when_the_port_is_configured(placeholder_value)
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with("http://${pipeline_id}:9201", placeholder_value)
+      }
+    end
+
+    data("in []"      => ["http://[${pipeline_id}]:9201", "2001:db8"],
+         "extra host" => ["http://[${pipeline_id}]:9201", "[2001:db8::1],[2001:db8::2]"],
+         "whole host" => ["http://${pipeline_id}:9201", "[2001:db8]"])
+    def test_rejects_placeholder_value_which_is_not_an_ipv6_address(data)
+      hosts, placeholder_value = data
+      assert_raise(Fluent::Plugin::OpenSearchOutput::UnrecoverableRequestFailure) {
+        expand_hosts_with(hosts, placeholder_value)
+      }
+    end
+
+    # The operator writes the structure, so only the placeholder value is
+    # checked. Every form that `get_connection_options` accepts keeps working.
+    data("plain host" => ["myhost-${pipeline_id},logs2.example.com:9201",
+                          "myhost-1,logs2.example.com:9201"],
+         "url"        => ["https://logs-${pipeline_id}.example.com:9201/os",
+                          "https://logs-1.example.com:9201/os"],
+         "userinfo"   => ["https://john:pass@logs-${pipeline_id}.example.com/os",
+                          "https://john:pass@logs-1.example.com/os"],
+         "ipv6"       => ["http://[2001:db8::1]:9201,myhost-${pipeline_id}",
+                          "http://[2001:db8::1]:9201,myhost-1"])
+    def test_keeps_the_configured_host_structure(data)
+      hosts, expected = data
+      assert_equal(expected, expand_hosts_with(hosts, "1"))
+    end
+
+    def test_warns_when_placeholder_host_is_used_with_credentials
+      d = driver(%{
+        host     logs-${tag}.example.com
+        user     john
+        password doe
+        @log_level info
+      })
+      assert_true(d.logs.any? { |log| log.include?("'host' uses a placeholder") })
+    end
+
+    def test_warns_when_hosts_embeds_credentials_in_front_of_a_placeholder
+      d = driver(%{
+        hosts https://john:secret@logs-${tag}.example.com/os
+        @log_level info
+      })
+      assert_true(d.logs.any? { |log| log.include?("in front of a host that a placeholder decides") })
+      assert_false(d.logs.any? { |log| log.include?("secret") })
+    end
+
+    def test_does_not_warn_when_credentials_belong_to_a_static_host
+      d = driver(%{
+        hosts https://john:secret@static.example.com/os,logs-${tag}.example.com
+        @log_level info
+      })
+      assert_false(d.logs.any? { |log| log.include?("in front of a host that a placeholder decides") })
+    end
   end
 
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
@@ -4117,4 +4405,43 @@ class OpenSearchOutputTest < Test::Unit::TestCase
     d.run
     assert { d.logs.any?(/\[error\]: Failed to get new AWS credentials: No valid AWS credentials found.\n/) }
   end
+
+  test 'a failed credential refresh keeps the current credentials and client' do
+    d, refresh = driver_with_captured_credential_refresh
+    instance = d.instance
+    credentials = instance.instance_variable_get(:@_aws_credentials)
+    instance.instance_variable_set(:@_os, :cached_client)
+    flexmock(instance).should_receive(:aws_credentials)
+      .and_raise(::RuntimeError.new("No valid AWS credentials found."))
+
+    refresh.call
+
+    assert_same credentials, instance.instance_variable_get(:@_aws_credentials)
+    assert_equal :cached_client, instance.instance_variable_get(:@_os)
+    assert { d.logs.any?(/Failed to get new AWS credentials: No valid AWS credentials found./) }
+  end
+
+  # The refresh shares @_os with the flush threads, so it has to use their lock
+  # instead of one of its own.
+  test 'a credential refresh swaps the credentials under the client lock' do
+    d, refresh = driver_with_captured_credential_refresh
+    instance = d.instance
+    new_credentials = Aws::Credentials.new('NEW_KEY', 'NEW_SECRET')
+    flexmock(instance).should_receive(:aws_credentials).and_return(new_credentials)
+    instance.instance_variable_set(:@_os, :cached_client)
+
+    # Records what the plugin had done by the time it left the lock.
+    seen = []
+    fake_lock = Object.new
+    fake_lock.define_singleton_method(:synchronize) do |&block|
+      block.call
+      seen << [instance.instance_variable_get(:@_aws_credentials),
+               instance.instance_variable_get(:@_os)]
+    end
+    instance.instance_variable_set(:@client_mutex, fake_lock)
+
+    refresh.call
+
+    assert_equal [[new_credentials, nil]], seen
+  end
 end

data/test/plugin/test_settings_template.json

@@ -0,0 +1,4 @@
+{
+  "template": "--index_prefix--*",
+  "settings": --settings--
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.8
 platform: ruby
 authors:
 - Hiroshi Hatake
@@ -249,6 +249,7 @@ files:
 - test/plugin/test_opensearch_tls.rb
 - test/plugin/test_out_opensearch.rb
 - test/plugin/test_out_opensearch_data_stream.rb
+- test/plugin/test_settings_template.json
 - test/plugin/test_template.json
 - test/test_log-ext.rb
 homepage: https://github.com/fluent/fluent-plugin-opensearch
@@ -270,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.18
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector
 test_files:
@@ -287,5 +288,6 @@ test_files:
 - test/plugin/test_opensearch_tls.rb
 - test/plugin/test_out_opensearch.rb
 - test/plugin/test_out_opensearch_data_stream.rb
+- test/plugin/test_settings_template.json
 - test/plugin/test_template.json
 - test/test_log-ext.rb