fluent-plugin-opensearch 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9dc42f6d13cb540c456050657cd2079a7b755a77e4148122f486be2b15486db7
-  data.tar.gz: 76bc650e96fc7b35a84641f6b0916024985bdccb779e54ef4e49d2172eb6227f
+  metadata.gz: 6382f3ad5be4b3c2c5aaa425568e4421c761bcb8cc5de65369123d01830a8888
+  data.tar.gz: 3e38aceba420423bf15065a0203ed6d696fa862d86d365a090d72d8da290a92a
 SHA512:
-  metadata.gz: dfa28860c87c55fcffba26a9d4990e64d909b8daf5e0ed68dfe309d6fcd3d78b61f41738bff3a88e6ffd780f1c870e0a95a151c5637c228f488253e2ac889a79
-  data.tar.gz: 77045d8ba592eaf9daf904b5d10ac36935c8d0ac13decff011e281cc0732e72f5b02ba69e196a8822a303dcb399752fe081d2bb846e0a19a93123513eac763e3
+  metadata.gz: 0bccfd9d20535f856b4dd115eb9126efc2cf2cfbe02d52fcb1f2e14db43747ee90de71f87873924540ea973f1d077d4618b87beb053e09e4d1c52d7b1b33bcc6
+  data.tar.gz: 3d9a7ce81fbb624f4af5e42dcfef10e4059073523a5b410af67575cffb11bf7e243f11d66cc6bf87851ea5e5dd0f2f7d76100923d53cf386df82ede3beb0c3ad

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 1.0.2
+ - Honor @hosts parameter for Data Streams (#21)
+ - Use template_file for Data Streams (#20)
+ - Specify host argument if needed (#11)
+
 ### 1.0.1
  -  Add testcases for hosts parameter (#10)
  - Permit to handle @hosts parameter (#9)

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.0.1'
+  s.version       = '1.0.2'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -29,16 +29,14 @@ module Fluent::Plugin
         @data_stream_names = []
       end
 
-      host = data_stream_connection
-
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
           retry_operate(@max_retry_putting_template,
                         @fail_on_putting_template_retry_exceed,
                         @catch_transport_exception_on_retry) do
-            create_index_template(@data_stream_name, @data_stream_template_name, host)
-            create_data_stream(@data_stream_name, host)
+            create_index_template(@data_stream_name, @data_stream_template_name)
+            create_data_stream(@data_stream_name)
           end
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
@@ -46,18 +44,9 @@ module Fluent::Plugin
       end
     end
 
-    # FIXME: Currently, the first element from hosts is only used and extracted.
-    def data_stream_connection
-      if host = get_connection_options[:hosts].first
-        "#{host[:scheme]}://#{host[:host]}:#{host[:port]}#{host[:path]}"
-      else
-        @host
-      end
-    end
-
     def validate_data_stream_parameters
       {"data_stream_name" => @data_stream_name,
-       "data_stream_template_name"=> @data_stream_template_name}.each do |parameter, value|
+       "data_stream_template_name" => @data_stream_template_name}.each do |parameter, value|
         unless valid_data_stream_parameters?(value)
           unless start_with_valid_characters?(value)
             if not_dots?(value)
@@ -80,19 +69,25 @@ module Fluent::Plugin
     end
 
     def create_index_template(datastream_name, template_name, host = nil)
-      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
-      body = {
-        "index_patterns" => ["#{datastream_name}*"],
-        "data_stream" => {},
-      }
-      params = {
-        name: template_name,
-        body: body
-      }
-      retry_operate(@max_retry_putting_template,
-                    @fail_on_putting_template_retry_exceed,
-                    @catch_transport_exception_on_retry) do
-        client(host).indices.put_index_template(params)
+      # Create index template from file
+      if @template_file
+        template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
+      else # Create default index template
+        return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+        body = {
+          "index_patterns" => ["#{datastream_name}*"],
+          "data_stream" => {},
+        }
+
+        params = {
+          name: template_name,
+          body: body
+        }
+        retry_operate(@max_retry_putting_template,
+                      @fail_on_putting_template_retry_exceed,
+                      @catch_transport_exception_on_retry) do
+          client(host).indices.put_index_template(params)
+        end
       end
     end
 
@@ -169,8 +164,13 @@ module Fluent::Plugin
     def write(chunk)
       data_stream_name = @data_stream_name
       data_stream_template_name = @data_stream_template_name
-      host = @host
+      host = nil
       if @use_placeholder
+        host = if @hosts
+                 extract_placeholders(@hosts, chunk)
+               else
+                 extract_placeholders(@host, chunk)
+               end
         data_stream_name = extract_placeholders(@data_stream_name, chunk)
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
         unless @data_stream_names.include?(data_stream_name)

data/test/plugin/datastream_template.json

@@ -0,0 +1,4 @@
+{
+  "index_patterns": ["foo*"],
+  "data_stream": {}
+}

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -118,6 +118,18 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     stub_data_stream(datastream_name)
   end
 
+  def stub_opensearch_with_store_index_command_counts(url="http://localhost:9200/_bulk")
+    if @index_command_counts == nil
+      @index_command_counts = {}
+      @index_command_counts.default = 0
+    end
+
+    stub_request(:post, url).with do |req|
+      index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_command_counts[url] += index_cmds.size
+    end
+  end
+
   # ref. https://opensearch.org/docs/latest/opensearch/data-streams/
   class DataStreamNameTest < self
 
@@ -329,9 +341,9 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
       data_stream_name default
     }
     stub_opensearch_info("https://host1:443/elastic//", "1.2.2",
-                         {'Authorization'=>'Basic am9objpwYXNzd29yZA=='})
+                         {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
     stub_opensearch_info("http://host2/default_path/_data_stream/default", "1.2.2",
-                         {'Authorization'=>'Basic am9objpwYXNzd29yZA=='})
+                         {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
     stub_existent_data_stream?("default", "https://host1/elastic/")
     instance = driver(config).instance
 
@@ -367,6 +379,19 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
 
+  def test_template_file
+    stub_default
+    cwd = File.dirname(__FILE__)
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => "foo_tpl",
+      'template_file' => File.join(cwd, 'datastream_template.json')
+    })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+  end
+
   def test_existent_data_stream
     stub_index_template
     stub_existent_data_stream?
@@ -484,13 +509,56 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
         '@type' => OPENSEARCH_DATA_STREAM_TYPE,
         'data_stream_name' => 'foo',
         'data_stream_template_name' => 'foo_tpl'
-      })
+    })
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
     assert_equal 1, @bulk_records
   end
 
+  def test_placeholder_writes_to_multi_hosts
+    stub_default("foo_bar", "foo_tpl_bar")
+    hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
+    hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
+    hosts.each do |host_info|
+      host, port = host_info
+      stub_opensearch_with_store_index_command_counts("http://#{host}:#{port}/foo_bar/_bulk")
+      stub_opensearch_info("http://#{host}:#{port}/")
+      stub_request(:get, "http://#{host}:#{port}/_data_stream/foo_bar").
+        to_return(status: 200, body: "", headers: {})
+    end
+
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo_${key1}',
+      'data_stream_template_name' => 'foo_tpl_${key1}',
+      'hosts' => "#{hosts_string}"
+    }, [config_element('buffer', 'tag,key1', {
+      'timekey' => '1d'
+    }, [])])
+    driver(conf).run(default_tag: 'test') do
+      hashes = {
+        'age' => rand(100),
+        'key1' => 'bar'
+      }
+      1000.times do
+        driver.feed(sample_record.merge(hashes))
+      end
+    end
+
+    # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
+    # it's Fluentd test driver's constraint
+    # so @index_command_counts.size is always 1
+    assert(@index_command_counts.size > 0, "not working with hosts options")
+
+    total = 0
+    @index_command_counts.each do |_, count|
+      total += count
+    end
+    assert_equal(2000, total)
+  end
+
   def test_template_retry_install_fails
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_index_template.json')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-31 00:00:00.000000000 Z
+date: 2022-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -208,6 +208,7 @@ files:
 - lib/fluent/plugin/out_opensearch.rb
 - lib/fluent/plugin/out_opensearch_data_stream.rb
 - test/helper.rb
+- test/plugin/datastream_template.json
 - test/plugin/test_alias_template.json
 - test/plugin/test_filter_opensearch_genid.rb
 - test/plugin/test_in_opensearch.rb
@@ -247,6 +248,7 @@ specification_version: 4
 summary: Opensearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
+- test/plugin/datastream_template.json
 - test/plugin/test_alias_template.json
 - test/plugin/test_filter_opensearch_genid.rb
 - test/plugin/test_in_opensearch.rb