fluent-plugin-opensearch 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56dc0da995d5f55c59e13afd75452b46e824f8337de6b074655dc8c92926eba8
-  data.tar.gz: dffab1c953b558fe1001df5c59844f1d71f8a2847199afade1b74ff9a05f739a
+  metadata.gz: 9dc42f6d13cb540c456050657cd2079a7b755a77e4148122f486be2b15486db7
+  data.tar.gz: 76bc650e96fc7b35a84641f6b0916024985bdccb779e54ef4e49d2172eb6227f
 SHA512:
-  metadata.gz: da67be6908b8000b6904655b0492cfedbedddae45dd7b408a41698e7c02ea1778b1fb8c64be4097b5cbfed1e58601c30a57892d07706de4782526cb0a6edd0c4
-  data.tar.gz: f002f9ccec9f8dd7c292ef3d88d8425e0f05e3651cf831eb86204059726643db43502c3285b4ad8410840e7bb29d814f2e243c1b6671c24909abb1f1b7b14824
+  metadata.gz: dfa28860c87c55fcffba26a9d4990e64d909b8daf5e0ed68dfe309d6fcd3d78b61f41738bff3a88e6ffd780f1c870e0a95a151c5637c228f488253e2ac889a79
+  data.tar.gz: 77045d8ba592eaf9daf904b5d10ac36935c8d0ac13decff011e281cc0732e72f5b02ba69e196a8822a303dcb399752fe081d2bb846e0a19a93123513eac763e3

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,37 +1,29 @@
 ---
 name:  Bug Report
-about: Create a report to help us improve. If you have questions about ES plugin on kubernetes, please direct these to https://discuss.kubernetes.io/ before sumbit kubernetes related issue.
+about: Create a report to help us improve. If you have questions about OpenSearch plugin on kubernetes, please direct these to https://discuss.kubernetes.io/ before sumbit kubernetes related issue.
 
 ---
 
 (check apply)
-- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+- [ ] read [the contribution guideline](https://github.com/fluent/fluent-plugin-opensearch/blob/master/CONTRIBUTING.md)
 - [ ] (optional) already reported 3rd party upstream repository or mailing list if you use k8s addon or helm charts.
 
-#### Problem
-
-...
-
 #### Steps to replicate
 
-Either clone and modify https://gist.github.com/pitr/9a518e840db58f435911
-
-**OR**
-
 Provide example config and message
 
 #### Expected Behavior or What you need to ask
 
 ...
 
-#### Using Fluentd and ES plugin versions
+#### Using Fluentd and OpenSearch plugin versions
 
 * OS version
 * Bare Metal or within Docker or Kubernetes or others?
-* Fluentd v0.12 or v0.14/v1.0
+* Fluentd v1.0 or later
   * paste result of ``fluentd --version`` or ``td-agent --version``
-* ES plugin 3.x.y/2.x.y or 1.x.y
+* OpenSearch plugin version
   * paste boot log of fluentd or td-agent
   * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
-* ES version (optional)
-* ES template(s) (optional)
+* OpenSearch version (optional)
+* OpenSearch template(s) (optional)

data/History.md

@@ -2,5 +2,10 @@
 
 ### [Unreleased]
 
+### 1.0.1
+ -  Add testcases for hosts parameter (#10)
+ - Permit to handle @hosts parameter (#9)
+ - Retry creating data stream/template when OpenSearch is unreachable (#8)
+
 ### 1.0.0
  - Initial public gem release.

data/README.md

@@ -6,7 +6,7 @@
 ![Testing on Ubuntu](https://github.com/fluent/fluent-plugin-opensearch/workflows/Testing%20on%20Ubuntu/badge.svg?branch=main)
 [![Coverage Status](https://coveralls.io/repos/github/fluent/fluent-plugin-opensearch/badge.svg?branch=upload-coverage-into-coveralls)](https://coveralls.io/github/fluent/fluent-plugin-opensearch?branch=main)
 
-Send your logs to OpenSearch (and search them with OpenSearch Dashboard maybe?)
+Send your logs to OpenSearch (and search them with OpenSearch Dashboards maybe?)
 
 * [Installation](#installation)
 * [Usage](#usage)

data/fluent-plugin-opensearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-opensearch'
-  s.version       = '1.0.0'
+  s.version       = '1.0.1'
   s.authors       = ['Hiroshi Hatake']
   s.email         = ['cosmo0920.wp@gmail.com']
   s.description   = %q{Opensearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -29,18 +29,32 @@ module Fluent::Plugin
         @data_stream_names = []
       end
 
-      @client = client
+      host = data_stream_connection
+
       unless @use_placeholder
         begin
           @data_stream_names = [@data_stream_name]
-          create_index_template(@data_stream_name, @data_stream_template_name, @host)
-          create_data_stream(@data_stream_name)
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
+            create_index_template(@data_stream_name, @data_stream_template_name, host)
+            create_data_stream(@data_stream_name, host)
+          end
         rescue => e
           raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
         end
       end
     end
 
+    # FIXME: Currently, the first element from hosts is only used and extracted.
+    def data_stream_connection
+      if host = get_connection_options[:hosts].first
+        "#{host[:scheme]}://#{host[:host]}:#{host[:port]}#{host[:path]}"
+      else
+        @host
+      end
+    end
+
     def validate_data_stream_parameters
       {"data_stream_name" => @data_stream_name,
        "data_stream_template_name"=> @data_stream_template_name}.each do |parameter, value|
@@ -65,7 +79,7 @@ module Fluent::Plugin
       end
     end
 
-    def create_index_template(datastream_name, template_name, host)
+    def create_index_template(datastream_name, template_name, host = nil)
       return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
       body = {
         "index_patterns" => ["#{datastream_name}*"],
@@ -78,17 +92,17 @@ module Fluent::Plugin
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        @client.indices.put_index_template(params)
+        client(host).indices.put_index_template(params)
       end
     end
 
-    def data_stream_exist?(datastream_name)
+    def data_stream_exist?(datastream_name, host = nil)
       params = {
         name: datastream_name
       }
       begin
         # TODO: Use X-Pack equivalent performing DataStream operation method on the following line
-        response = @client.perform_request('GET', "/_data_stream/#{datastream_name}", {}, params)
+        response = client(host).perform_request('GET', "/_data_stream/#{datastream_name}", {}, params)
         return (not response.is_a?(OpenSearch::Transport::Transport::Errors::NotFound))
       rescue OpenSearch::Transport::Transport::Errors::NotFound => e
         log.info "Specified data stream does not exist. Will be created: <#{e}>"
@@ -96,8 +110,8 @@ module Fluent::Plugin
       end
     end
 
-    def create_data_stream(datastream_name)
-      return if data_stream_exist?(datastream_name)
+    def create_data_stream(datastream_name, host = nil)
+      return if data_stream_exist?(datastream_name, host)
       params = {
         name: datastream_name
       }
@@ -105,7 +119,7 @@ module Fluent::Plugin
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
         # TODO: Use X-Pack equivalent performing DataStream operation method on the following line
-        @client.perform_request('PUT', "/_data_stream/#{datastream_name}", {}, params)
+        client(host).perform_request('PUT', "/_data_stream/#{datastream_name}", {}, params)
       end
     end
 
@@ -162,7 +176,7 @@ module Fluent::Plugin
         unless @data_stream_names.include?(data_stream_name)
           begin
             create_index_template(data_stream_name, data_stream_template_name, host)
-            create_data_stream(data_stream_name)
+            create_data_stream(data_stream_name, host)
             @data_stream_names << data_stream_name
           rescue => e
             raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
@@ -191,7 +205,7 @@ module Fluent::Plugin
         body: bulk_message
       }
       begin
-        response = @client.bulk(params)
+        response = client(host).bulk(params)
         if response['errors']
           log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
         end

data/test/plugin/test_out_opensearch_data_stream.rb

@@ -61,38 +61,43 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
   DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
 
-  def stub_index_template(name="foo_tpl")
-    stub_request(:put, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_index_template(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:put, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_data_stream(name="foo")
-    stub_request(:put, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_data_stream(name="foo", url="localhost:9200")
+    stub_request(:put, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_data_stream?(name="foo")
-    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_data_stream?(name="foo", url="localhost:9200")
+    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_existent_template?(name="foo_tpl")
-    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+  def stub_existent_template?(name="foo_tpl", url="localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
-  def stub_nonexistent_data_stream?(name="foo")
-    stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, OpenSearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_data_stream?(name="foo", url="localhost:9200")
+    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, OpenSearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_nonexistent_template?(name="foo_tpl")
-    stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [404, OpenSearch::Transport::Transport::Errors::NotFound])
+  def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, OpenSearch::Transport::Transport::Errors::NotFound])
   end
 
-  def stub_bulk_feed(datastream_name="foo", template_name="foo_tpl")
-    stub_request(:post, "http://localhost:9200/#{datastream_name}/_bulk").with do |req|
+  def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
+    stub_request(:get, "#{url}/_index_template/#{name}").
+      to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
+  end
+
+  def stub_bulk_feed(datastream_name="foo", template_name="foo_tpl", url="http://localhost:9200")
+    stub_request(:post, "#{url}/#{datastream_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_data_stream/foo_bar
       # {"@timestamp": ...}
       @bulk_records += req.body.split("\n").size / 2
     end
-    stub_request(:post, "http://localhost:9200/#{template_name}/_bulk").with do |req|
+    stub_request(:post, "http://#{url}#{template_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_data_stream/foo_bar
       # {"@timestamp": ...}
@@ -100,13 +105,13 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="1.2.2")
+  def stub_opensearch_info(url="http://localhost:9200/", version="1.2.2", headers={})
     body ="{\"version\":{\"number\":\"#{version}\", \"distribution\":\"opensearch\"},\"tagline\":\"The OpenSearch Project: https://opensearch.org/\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' }.merge(headers) })
   end
 
   def stub_default(datastream_name="foo", template_name="foo_tpl", host="http://localhost:9200")
-    stub_elastic_info(host)
+    stub_opensearch_info(host)
     stub_nonexistent_template?(template_name)
     stub_index_template(template_name)
     stub_nonexistent_data_stream?(datastream_name)
@@ -315,11 +320,58 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
 
+  def test_hosts_list_configure
+    config = %{
+      hosts            https://john:password@host1:443/elastic/,http://host2
+      path             /default_path
+      user             default_user
+      password         default_password
+      data_stream_name default
+    }
+    stub_opensearch_info("https://host1:443/elastic//", "1.2.2",
+                         {'Authorization'=>'Basic am9objpwYXNzd29yZA=='})
+    stub_opensearch_info("http://host2/default_path/_data_stream/default", "1.2.2",
+                         {'Authorization'=>'Basic am9objpwYXNzd29yZA=='})
+    stub_existent_data_stream?("default", "https://host1/elastic/")
+    instance = driver(config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'john', host1[:user]
+    assert_equal 'password', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
+  def test_datastream_configure_retry
+    stub_opensearch_info
+    stub_nonexistent_template_retry?
+    stub_index_template
+    stub_nonexistent_data_stream?
+    stub_data_stream
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => OPENSEARCH_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_template_name' => "foo_tpl"
+    })
+    assert_equal "foo", driver(conf).instance.data_stream_name
+  end
+
   def test_existent_data_stream
     stub_index_template
     stub_existent_data_stream?
     stub_data_stream
-    stub_elastic_info
+    stub_opensearch_info
     conf = config_element(
       'ROOT', '', {
         '@type' => OPENSEARCH_DATA_STREAM_TYPE,
@@ -333,7 +385,7 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
     stub_index_template
     stub_existent_data_stream?
     stub_data_stream
-    stub_elastic_info
+    stub_opensearch_info
     conf = config_element(
       'ROOT', '', {
         '@type' => OPENSEARCH_DATA_STREAM_TYPE,
@@ -463,7 +515,7 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
       connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
     end
-    stub_elastic_info("https://logs.google.com:778/")
+    stub_opensearch_info("https://logs.google.com:778/")
 
     assert_raise(Fluent::Plugin::OpenSearchError::RetryableOperationExhaustedFailure) do
       driver(config)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Hiroshi Hatake
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-13 00:00:00.000000000 Z
+date: 2022-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -226,7 +226,7 @@ licenses:
 - Apache-2.0
 metadata:
   changelog_uri: https://github.com/fluent/fluent-plugin-opensearch/blob/master/History.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -241,8 +241,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key: 
+rubygems_version: 3.2.30
+signing_key:
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector
 test_files: