fluent-plugin-input-opensearch 1.2.4 → 1.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9418f867988b41854b21285006db5adce7875d9a0eb5e480989e70146674fddd
-  data.tar.gz: fc1aaa976dffc952a028dc944048cf579d8bd0112003ea40ab82ca26149a2d1d
+  metadata.gz: 20c770361eae05639dd05467514f4d5d725ed081fa6f9316391bfe2d4e410b6e
+  data.tar.gz: 4af0f71d3bd9df224252232192bc3d630df4818c23870705bf36f4a98507b422
 SHA512:
-  metadata.gz: 61cbf7e514a16a3e292d072f59d81bc5d37ab6aa7e6d8bc5a8a2ec76d1cbc7972054e109b003c39be1d2195ff6c9648d153c62f72c708e7a4d999c3489f2bf08
-  data.tar.gz: 71484682650b25de3a6e77ae3e17621c2a8ff691e80ea1e6ed9f5a809f658eee8f2122271f7b8aa8919b73076c2afc08d50f70659bab2c67babc53bdec7506b2
+  metadata.gz: 71b0e17756425c15b2b3f5d4b0ab926785f9c8730304cac2d871a70bc396b30d662135c7f235c44fc92b398be212a4d4f430e16a9472db6d07202d9f9aa20cc6
+  data.tar.gz: 4b4f75328bf00643d93d937a9fc5ee479582019761ec1e4de64794faa57a68fe5d823d28e053d02e71fdd0801d1b9fd9f5087ee9a5482a524b62fe4a5dbe6c88

data/README.md

@@ -605,10 +605,10 @@ request_timeout 15s # defaults to 5s
 
 ### reload_connections
 
-You can tune how the opensearch-transport host reloading feature works. By default it will reload the host list from the server every 10,000th request to spread the load. This can be an issue if your OpenSearch cluster is behind a Reverse Proxy, as Fluentd process may not have direct network access to the OpenSearch nodes.
+You can tune how the opensearch-transport host reloading feature works. By default it off but if you set true it will reload the host list from the server every 10,000th request to spread the load. This can be an issue if your OpenSearch cluster is behind a Reverse Proxy, as Fluentd process may not have direct network access to the OpenSearch nodes.
 
 ```
-reload_connections false # defaults to true
+reload_connections false # defaults to false
 ```
 
 ### reload_on_failure

data/lib/fluent/plugin/in_opensearch.rb

@@ -344,8 +344,8 @@ module Fluent::Plugin
         end
         @retry.step
         if error.message.include?('EOFError (EOFError)')
-          log.error("Restart plugin #{error.message}")
-          exit(1)
+          log.error("Restart plugin because hit error #{error.message}")
+          start
         end
         #Raise error if the retry limit has been reached
         raise "Hit limit for retries. retry_times: #{@retry.steps}, error: #{error.message}" if @retry.limit?
@@ -370,7 +370,7 @@ module Fluent::Plugin
           run_slice(slice_id)
         end
       end
-    rescue Faraday::ConnectionFailed, OpenSearch::Transport::Transport::Error => error
+    rescue => error
       update_retry_state(error)
       retry
     end

data/lib/fluent/plugin/out_opensearch.rb

@@ -350,7 +350,11 @@ module Fluent::Plugin
 
             @credential_mutex.synchronize do
               @_os = nil
-              @_aws_credentials = aws_credentials(@endpoint)
+              begin
+                @_aws_credentials = aws_credentials(@endpoint)
+              rescue => e
+                log.error("Failed to get new AWS credentials: #{e}")
+              end
             end
           end
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-input-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.2.5
 platform: ruby
 authors:
 - imcotop
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-25 00:00:00.000000000 Z
+date: 2025-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: '1.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -199,28 +199,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".coveralls.yml"
-- ".editorconfig"
-- ".github/ISSUE_TEMPLATE/bug_report.md"
-- ".github/ISSUE_TEMPLATE/feature_request.md"
-- ".github/PULL_REQUEST_TEMPLATE/pull_request_template.md"
-- ".github/workflows/coverage.yaml"
-- ".github/workflows/issue-auto-closer.yml"
-- ".github/workflows/linux.yml"
-- ".github/workflows/macos.yml"
-- ".github/workflows/windows.yml"
-- ".gitignore"
-- CONTRIBUTING.md
-- Gemfile
-- History.md
 - LICENSE.txt
-- README.OpenSearchGenID.md
-- README.OpenSearchInput.md
-- README.Troubleshooting.md
 - README.md
-- Rakefile
-- fluent-plugin-opensearch.gemspec
-- gemfiles/Gemfile.elasticsearch.v6
 - lib/fluent/log-ext.rb
 - lib/fluent/plugin/filter_opensearch_genid.rb
 - lib/fluent/plugin/in_opensearch.rb
@@ -234,21 +214,6 @@ files:
 - lib/fluent/plugin/opensearch_tls.rb
 - lib/fluent/plugin/out_opensearch.rb
 - lib/fluent/plugin/out_opensearch_data_stream.rb
-- test/helper.rb
-- test/plugin/datastream_template.json
-- test/plugin/test_alias_template.json
-- test/plugin/test_filter_opensearch_genid.rb
-- test/plugin/test_in_opensearch.rb
-- test/plugin/test_index_alias_template.json
-- test/plugin/test_index_template.json
-- test/plugin/test_oj_serializer.rb
-- test/plugin/test_opensearch_error_handler.rb
-- test/plugin/test_opensearch_fallback_selector.rb
-- test/plugin/test_opensearch_tls.rb
-- test/plugin/test_out_opensearch.rb
-- test/plugin/test_out_opensearch_data_stream.rb
-- test/plugin/test_template.json
-- test/test_log-ext.rb
 homepage: https://github.com/fluent/fluent-plugin-opensearch
 licenses:
 - Apache-2.0
@@ -273,19 +238,4 @@ rubygems_version: 3.5.7
 signing_key:
 specification_version: 4
 summary: Opensearch output plugin for Fluent event collector
-test_files:
-- test/helper.rb
-- test/plugin/datastream_template.json
-- test/plugin/test_alias_template.json
-- test/plugin/test_filter_opensearch_genid.rb
-- test/plugin/test_in_opensearch.rb
-- test/plugin/test_index_alias_template.json
-- test/plugin/test_index_template.json
-- test/plugin/test_oj_serializer.rb
-- test/plugin/test_opensearch_error_handler.rb
-- test/plugin/test_opensearch_fallback_selector.rb
-- test/plugin/test_opensearch_tls.rb
-- test/plugin/test_out_opensearch.rb
-- test/plugin/test_out_opensearch_data_stream.rb
-- test/plugin/test_template.json
-- test/test_log-ext.rb
+test_files: []

data/.coveralls.yml

@@ -1 +0,0 @@
-service_name: github-ci

data/.editorconfig

@@ -1,9 +0,0 @@
-# EditorConfig: http://EditorConfig.org
-
-root = true
-
-[*]
-end_of_line = lf
-insert_final_newline = true
-indent_style = space
-indent_size = 2

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,29 +0,0 @@
----
-name:  Bug Report
-about: Create a report to help us improve. If you have questions about OpenSearch plugin on kubernetes, please direct these to https://discuss.kubernetes.io/ before sumbit kubernetes related issue.
-
----
-
-(check apply)
-- [ ] read [the contribution guideline](https://github.com/fluent/fluent-plugin-opensearch/blob/master/CONTRIBUTING.md)
-- [ ] (optional) already reported 3rd party upstream repository or mailing list if you use k8s addon or helm charts.
-
-#### Steps to replicate
-
-Provide example config and message
-
-#### Expected Behavior or What you need to ask
-
-...
-
-#### Using Fluentd and OpenSearch plugin versions
-
-* OS version
-* Bare Metal or within Docker or Kubernetes or others?
-* Fluentd v1.0 or later
-  * paste result of ``fluentd --version`` or ``td-agent --version``
-* OpenSearch plugin version
-  * paste boot log of fluentd or td-agent
-  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
-* OpenSearch version (optional)
-* OpenSearch template(s) (optional)

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,24 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-
----
-
-(check apply)
-- [ ] read [the contribution guideline](https://github.com/fluent/fluent-plugin-opensearch/blob/master/CONTRIBUTING.md)
-
-**Is your feature request related to a problem? Please describe.**
-
-<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
-
-**Describe the solution you'd like**
-
-<!-- A clear and concise description of what you want to happen. -->
-
-**Describe alternatives you've considered**
-
-<!-- A clear and concise description of any alternative solutions or features you've considered. -->
-
-**Additional context**
-
-<!-- Add any other context or screenshots about the feature request here. -->

data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -1,9 +0,0 @@
-DESCRIPTION HERE
-
-(check all that apply)
-- [ ] tests added
-- [ ] tests passing
-- [ ] README updated (if needed)
-- [ ] README Table of Contents updated (if needed)
-- [ ] History.md and `version` in gemspec are untouched
-- [ ] backward compatible

data/.github/workflows/coverage.yaml

@@ -1,22 +0,0 @@
-name: coverage
-
-on: [push, pull_request]
-
-jobs:
-  coverage:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 3.0
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '3.1'
-      - name: Build and test with Rake
-        run: |
-          gem install bundler
-          bundle install
-          bundle exec rake
-      - name: Coveralls GitHub Action
-        uses: coverallsapp/github-action@v1.1.2
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}

data/.github/workflows/issue-auto-closer.yml

@@ -1,12 +0,0 @@
-name: Autocloser
-on: [issues]
-jobs:
-  autoclose:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Autoclose issues that did not follow issue template
-      uses: roots/issue-closer-action@v1.1
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
-        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)|(.*Is your feature request related to a problem? Please describe.*)|(.*Describe the solution you'd like.*)|(.*Describe alternatives you've considered.*)"

data/.github/workflows/linux.yml

@@ -1,26 +0,0 @@
-name: Testing on Ubuntu
-on:
-  - push
-  - pull_request
-jobs:
-  build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
-        os:
-          - ubuntu-latest
-    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-    - name: unit testing
-      env:
-        CI: true
-      run: |
-        gem install bundler rake
-        bundle install --jobs 4 --retry 3
-        bundle exec rake test

data/.github/workflows/macos.yml

@@ -1,26 +0,0 @@
-name: Testing on macOS
-on:
-  - push
-  - pull_request
-jobs:
-  build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
-        os:
-          - macOS-latest
-    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-    - name: unit testing
-      env:
-        CI: true
-      run: |
-        gem install bundler rake
-        bundle install --jobs 4 --retry 3
-        bundle exec rake test

data/.github/workflows/windows.yml

@@ -1,26 +0,0 @@
-name: Testing on Windows
-on:
-  - push
-  - pull_request
-jobs:
-  build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
-        os:
-          - windows-latest
-    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-    - name: unit testing
-      env:
-        CI: true
-      run: |
-        gem install bundler rake
-        bundle install --jobs 4 --retry 3
-        bundle exec rake test

data/.gitignore

@@ -1,18 +0,0 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp
-.DS_Store

data/CONTRIBUTING.md

@@ -1,24 +0,0 @@
-Contributing to fluent-plugin-elasticsearch
-===
-
-We'd love your contribution. Here are the guidelines!
-
-## Got a question or problem?
-
-RESOURCES of [OpenSearch plugin documentation](https://github.com/fluent/fluent-plugin-opensearch/blob/master/README.md) may help you.
-
-If you have further questions about OpenSearch plugin, feel free to [open an Issue](https://github.com/fluent/fluent-plugin-opensearch/issues).
-
-## Found a bug?
-
-If you find a bug of OpenSearch plugin or a mistake in the documentation, you can help us by submitting an issue to OpenSearch plugin. Even better you can submit a pull request with a fix.
-
-Note: Please follow the issue template/pull request template when sending your issue. Otherwise, your issue _might going to be closed with no response_.
-
-## Patch Guidelines
-
-Here are some things that would increase a chance that your patch is accepted:
-
-* Write tests
-* Run tests before sending pull request with `bundle exec rake test`
-* Write [a good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)

data/Gemfile

@@ -1,10 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in fluent-plugin-opensearch.gemspec
-gemspec
-
-gem 'simplecov', require: false
-gem 'simplecov-lcov', require: false
-gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
-gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
-gem "oj"

data/History.md

@@ -1,67 +0,0 @@
-## Changelog [[tags]](https://github.com/fluent/fluent-plugin-opensearch/tags)
-
-### [Unreleased]
-
-### 1.1.4
- - test: remove minitest to correct misjudge of the framework by flexmock (#114)
- - Add logic to write method of out_opensearch_data_stream (#109)
-
-### 1.1.3
- - Revert the behavior of passing duration second (#108)
-
-### 1.1.2
- - Check OS cluster for data streams and templates for index template creation (#106)
- - out\_opensearch\_data\_stream: Don't connect to opensearch on dry-run (#105)
-
-### 1.1.1
- -  Pass a value of refresh\_credentials\_interval as duration\_seconds (#78)
-
-### 1.1.0
- - Unpin `faraday` from v1, upgrade to v2.
-   Note that if you can't migrate other plugins from `faraday` v1 yet, need to keep
-   fluent-plugin-opensearch v1.0.10.
-
-### 1.0.10
- - Replace File.exists? with File.exist? to work with Ruby 3.2 (#93)
- - Add a constraint for dependent gem to stay on Faraday v1 (#90)
- - README.md: Fix a link to opensearch-ruby (#85)
-
-### 1.0.9
- - Adjust GitHub workflows (#89)
- - out\_opensearch: Provide service_name choices for handling serverless (#88)
-
-### 1.0.8
- - Use faraday 1.x explicitly (#71)
-
-### 1.0.7
- - Expire AWS credentials with a certain interval (#52)
-
-### 1.0.6
- - out\_opensearch: Handle suppress\_type\_name operation correctly (#61)
-
-### 1.0.5
- -  Use if clause to detect requirements for emitting error label events (#57)
- - opensearch_data_stream: Align lowercases for data_stream and its template names (#50)
-
-### 1.0.4
- - Automatically create data streams (#44)
-
-### 1.0.3
- - Configurable unrecoverable record types (#40)
- - Handle exceptions on retrieving AWS credentials (#39)
- - Suppress emit error label events (#38)
- - Provide suppress_type_name parameter (#29)
- - Honor @time_key (data streams) (#28)
-
-### 1.0.2
- - Honor @hosts parameter for Data Streams (#21)
- - Use template_file for Data Streams (#20)
- - Specify host argument if needed (#11)
-
-### 1.0.1
- -  Add testcases for hosts parameter (#10)
- - Permit to handle @hosts parameter (#9)
- - Retry creating data stream/template when OpenSearch is unreachable (#8)
-
-### 1.0.0
- - Initial public gem release.

data/README.OpenSearchGenID.md

@@ -1,116 +0,0 @@
-## Index
-
-* [Usage](#usage)
-* [Configuration](#configuration)
-  + [hash_id_key](#hash_id_key)
-  + [include_tag_in_seed](#include_tag_in_seed)
-  + [include_time_in_seed](#include_time_in_seed)
-  + [use_record_as_seed](#use_record_as_seed)
-  + [use_entire_record](#use_entire_record)
-  + [record_keys](#record_keys)
-  + [separator](#separator)
-  + [hash_type](#hash_type)
-* [Advanced Usage](#advanced-usage)
-
-## Usage
-
-In your Fluentd configuration, use `@type opensearch_genid`. Additional configuration is optional, default values would look like this:
-
-```
-<filter>
-  @type opensearch_genid
-  hash_id_key _hash
-  include_tag_in_seed false
-  include_time_in_seed false
-  use_record_as_seed false
-  use_entire_record false
-  record_keys []
-  separator _
-  hash_type sha1
-</filter>
-```
-
-## Configuration
-
-### hash_id_key
-
-```
-hash_id_key _id
-```
-
-You can specify generated hash storing key.
-
-### include_tag_in_seed
-
-```
-include_tag_in_seed true
-```
-
-You can specify to use tag for hash generation seed.
-
-### include_time_in_seed
-
-```
-include_time_in_seed true
-```
-
-You can specify to use time for hash generation seed.
-
-### use_record_as_seed
-
-```
-use_record_as_seed true
-```
-
-You can specify to use record in events for hash generation seed. This parameter should be used with [record_keys](#record_keys) parameter in practice.
-
-### record_keys
-
-```
-record_keys request_id,pipeline_id
-```
-
-You can specify keys which are record in events for hash generation seed. This parameter should be used with [use_record_as_seed](#use_record_as_seed) parameter in practice.
-
-### use_entire_record
-
-```
-use_entire_record true
-```
-
-You can specify to use entire record in events for hash generation seed.
-
-
-### separator
-
-```
-separator |
-```
-
-You can specify separator charactor to creating seed for hash generation.
-
-### hash_type
-
-```
-hash_type sha1
-```
-
-You can specify hash algorithm. Support algorithms `md5`, `sha1`, `sha256`, `sha512`. Default: `sha1`
-
-## Advanced Usage
-
-Opensearch GenID plugin can handle record contents differing with the following parameters:
-
-```aconf
-<filter the.awesome.your.routing.tag>
-  @type opensearch_genid
-  use_entire_record true
-  hash_type sha1
-  hash_id_key _hash
-  separator _
-  inc_time_as_key true
-  inc_tag_as_key true
-</filter>
-```
-
-The above configuration can handle tag, time, and record differing and generate different base64 encoded hash per record.