fluent-plugin-opensearch 1.1.0 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecc83833c1b081c2c7a43eb518ec339c2fe2b1c3153ea424d02b046bfd26129c
-  data.tar.gz: '09443e78f21f9e3a064c0fdc0faea5d20d2b67ce85f8f187cef5322382a827e5'
+  metadata.gz: 578e5c63061d5f1cb3f3729dee4ea630564dc60d82f6b047c8e4b23830fdf16a
+  data.tar.gz: a502d46eb5a3a59c5c4f65f9009623ff32fde1de4cfda7e181437a8720bdbda1
 SHA512:
-  metadata.gz: fa530480b103bacbe4e00bc0bff3d3aca549acb126fc2251affa80267e24391bd929f9c9a903906c83807cdddc880a6a53b6470ec944fcaaecd73943bcb3560a
-  data.tar.gz: bc8a40ae17f566b1176edf1f163e8df740df72c19f99fce220dcc0a88cc1c857bfd5b1014f201f5261328eee234d3eb24a2421cc7b882e6c1b047e7e660fb4f3
+  metadata.gz: 77b3373041396440cf8debaea8fd2ab95b94df950cbfe6105f65d45cce7092a6f89bd019b38d4022051c651541d3d21918625ea1b9939f58108101eae90a027f
+  data.tar.gz: 2e746cd797f5d24f08dbb386ea42b9bff506831c56f9f4b92dcf99dde321b9b613c7473cbad39bd1f6d338ae6dd094a87f838141306111ab49b5a76886346918

data/History.md

@@ -2,6 +2,20 @@
 
 ### [Unreleased]
 
+### 1.1.4
+ - test: remove minitest to correct misjudge of the framework by flexmock (#114)
+ - Add logic to write method of out_opensearch_data_stream (#109)
+
+### 1.1.3
+ - Revert the behavior of passing duration second (#108)
+
+### 1.1.2
+ - Check OS cluster for data streams and templates for index template creation (#106)
+ - out\_opensearch\_data\_stream: Don't connect to opensearch on dry-run (#105)
+
+### 1.1.1
+ -  Pass a value of refresh\_credentials\_interval as duration\_seconds (#78)
+
 ### 1.1.0
  - Unpin `faraday` from v1, upgrade to v2.
    Note that if you can't migrate other plugins from `faraday` v1 yet, need to keep

data/README.md

@@ -1606,7 +1606,7 @@ There are usually a few feature requests, tagged [Easy](https://github.com/fluen
 
 Pull Requests are welcomed.
 
-Becore send a pull request or report an issue, please read [the contribution guideline](CONTRIBUTING.md).
+Before sending a pull request or reporting an issue, please read [the contribution guideline](CONTRIBUTING.md).
 
 ## Running tests
 

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.1.0'
+  s.version       = '1.1.4'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -69,23 +69,26 @@ module Fluent::Plugin
 
     def create_index_template(datastream_name, template_name, host = nil)
       # Create index template from file
-      if @template_file
-        template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
-      else # Create default index template
-        return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
-        body = {
-          "index_patterns" => ["#{datastream_name}*"],
-          "data_stream" => {},
-        }
-
-        params = {
-          name: template_name,
-          body: body
-        }
-        retry_operate(@max_retry_putting_template,
-                      @fail_on_putting_template_retry_exceed,
-                      @catch_transport_exception_on_retry) do
-          client(host).indices.put_index_template(params)
+      if !dry_run?
+        if @template_file
+          return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+          template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
+        else # Create default index template
+          return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+          body = {
+            "index_patterns" => ["#{datastream_name}*"],
+            "data_stream" => {},
+          }
+
+          params = {
+            name: template_name,
+            body: body
+          }
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
+            client(host).indices.put_index_template(params)
+          end
         end
       end
     end
@@ -159,13 +162,10 @@ module Fluent::Plugin
                end
         data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
         data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
-        unless @data_stream_names.include?(data_stream_name)
-          begin
-            create_index_template(data_stream_name, data_stream_template_name, host)
-            @data_stream_names << data_stream_name
-          rescue => e
-            raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
-          end
+        begin
+          create_index_template(data_stream_name, data_stream_template_name, host)
+        rescue => e
+          raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
         end
       end
 
@@ -187,6 +187,12 @@ module Fluent::Plugin
             dt = Time.at(time).to_datetime
           end
           record.merge!({"@timestamp" => dt.iso8601(@time_precision)})
+          if @include_tag_key
+            record[@tag_key] = tag
+          end
+          if @remove_keys
+            @remove_keys.each { |key| record.delete(key) }
+          end
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
         rescue => e
           emit_error_label_event do

data/test/helper.rb

@@ -55,7 +55,6 @@ end
 
 require 'test/unit'
 require 'fluent/test'
-require 'minitest/pride'
 
 require 'webmock/test_unit'
 WebMock.disable_net_connect!

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -678,4 +678,69 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     assert(index_cmds[1].has_key? '@timestamp')
   end
 
+  def test_record_with_include_tag_key
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'include_tag_key' => true,
+      'tag_key' => 'test_tag'
+    })
+    record = {
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(index_cmds[1].has_key?('test_tag'))
+  end
+
+  def test_record_without_include_tag_key
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'include_tag_key' => false
+    })
+    record = {
+      'message' => 'Sample Record'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(!index_cmds[1].has_key?('test'))
+  end
+
+  def test_record_with_remove_keys
+    stub_default
+    stub_bulk_feed
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => 'foo_tpl',
+      'remove_keys' => 'remove_me'
+    })
+    record = {
+      'message' => 'Sample Record',
+      'remove_me' => 'foo'
+    }
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(record)
+    end
+    assert(!index_cmds[1].has_key?('remove_me'))
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.4
 platform: ruby
 authors:
 - Hiroshi Hatake
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-11 00:00:00.000000000 Z
+date: 2023-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -270,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector