logstash-output-opensearch 1.3.0-java → 2.0.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbe40e1f039f2d91a4adfb8b8fb9a28fd877156bf270060fa1d294986a769983
-  data.tar.gz: 9c2bd44c3e32cbf31d3ffac173a5e9b0ff2e4bcb9c4bd27267ac70a3a88ee6da
+  metadata.gz: a9700db92f60b264d7edf26bb34281f9fac13c27d93564d80ce007e5e7b73928
+  data.tar.gz: a4feb8d63b747cd2db45ec1b47df5e30a6aea6dbee3a30dde51c94773b192ec8
 SHA512:
-  metadata.gz: ed114d31a8dd15ebe57cba0d87c278efe3be2a47743aa2cdfe99bf861e0934edb2772d09aa59eda0d9428b119dde77a16c82371ce19ca1cbfb5c6ce736692690
-  data.tar.gz: df6263f5109d5834bc19fa9804c082c95dc1867eed7926be99b7682c4c1cfb3abffa0ff23b16f123f55f196e353527d31d8d5a61f3011bbf56805f43fa890fdc
+  metadata.gz: a7ad196b7d23790cf9bed976a3c51b0a2ebbd2d1bb9fa92617a9994620d0be287c142c5ce944cb941626f598491d38b112d68797f7953e63180204522a639af3
+  data.tar.gz: cdd308cd216f67382e466e98546f04e6ac1a011e6d4f64e4b27e23d44809ae6a069c3849e62986a0f79ccb26d40eaa12fb998271072efaf87be1a5ca765da48a

data/COMPATIBILITY.md

@@ -5,23 +5,25 @@
 |  logstash-output-opensearch | Logstash OSS|
 | ------------- | ------------- |
 | 1.0.0+  | 7.13.2  |
+| 2.0.0+  | 7.13.2  |
 
 ### Matrix for OpenSearch
 
 |  logstash-output-opensearch | OpenSearch  |
 | ------------- | ------------- |
-| 1.0.0+  | 1.0.0  |  
-
+| 1.0.0+  | 1.0.0  |
+| 2.0.0+  | 1.0.0  |
 
 ### Matrix for ODFE
 
 |  logstash-output-opensearch |  ODFE |
 | ------------- | ------------- |
 | 1.0.0+   |   1.x - 1.13.2 |
-
+| 2.0.0+   |   1.x - 1.13.2 |
 
 ### Matrix for Elasticsearch OSS
 
 |  logstash-output-opensearch |  Elasticsearch OSS|
 | ------------- | ------------- |
 | 1.0.0+   |  7.x - 7.10.2      |
+| 2.0.0+   |  7.x - 7.10.2      |

data/MAINTAINERS.md

@@ -1,74 +1,24 @@
-- [Overview](#overview)
-- [Current Maintainers](#current-maintainers)
-- [Maintainer Responsibilities](#maintainer-responsibilities)
-    - [Uphold Code of Conduct](#uphold-code-of-conduct)
-    - [Prioritize Security](#prioritize-security)
-    - [Review Pull Requests](#review-pull-requests)
-    - [Triage Open Issues](#triage-open-issues)
-    - [Be Responsive](#be-responsive)
-    - [Maintain Overall Health of the Repo](#maintain-overall-health-of-the-repo)
-    - [Use Semver](#use-semver)
-    - [Release Frequently](#release-frequently)
-    - [Promote Other Maintainers](#promote-other-maintainers)
-
 ## Overview
 
-This document explains who the maintainers are (see below), what they do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).
+This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Current Maintainers
 
-| Maintainer               | GitHub ID                               | Affiliation |
-| ------------------------ | --------------------------------------- | ----------- |
-| Jack Mazanec             | [jmazanec15](https://github.com/jmazanec15) | Amazon |
-| Vamshi Vijay Nakkirtha   | [vamshin](https://github.com/vamshin)   |   Amazon    |
-| Vijayan Balasubramanian  | [VijayanB](https://github.com/VijayanB) |   Amazon    |
-| Deep Datta               | [deepdatta](https://github.com/deepdatta) | Amazon    |
-| David Venable            | [dlvenable](https://github.com/dlvenable) | Amazon    |
-| Shivani Shukla           | [sshivanii](https://github.com/sshivanii) | Amazon    |
-
-## Maintainer Responsibilities
-
-Maintainers are active and visible members of the community, and have [maintain-level permissions on a repository](https://docs.github.com/en/organizations/managing-access-to-your-organizations-repositories/repository-permission-levels-for-an-organization). Use those privileges to serve the community and evolve code as follows.
-
-### Uphold Code of Conduct
-
-Model the behavior set forward by the [Code of Conduct](CODE_OF_CONDUCT.md) and raise any violations to other maintainers and admins.
-
-### Prioritize Security
-
-Security is your number one priority. Maintainer's Github keys must be password protected securely and any reported security vulnerabilities are addressed before features or bugs.
-
-Note that this repository is monitored and supported 24/7 by Amazon Security, see [Reporting a Vulnerability](SECURITY.md) for details.
-
-### Review Pull Requests
-
-Review pull requests regularly, comment, suggest, reject, merge and close. Accept only high quality pull-requests. Provide code reviews and guidance on incomming pull requests. Don't let PRs be stale and do your best to be helpful to contributors.
-
-### Triage Open Issues
-
-Manage labels, review issues regularly, and triage by labelling them.
-
-All repositories in this organization have a standard set of labels, including `bug`, `documentation`, `duplicate`, `enhancement`, `good first issue`, `help wanted`, `blocker`, `invalid`, `question`, `wontfix`, and `untriaged`, along with release labels, such as `v1.0.0`, `v1.1.0` and `v2.0.0`, and `backport`.
-
-Use labels to target an issue or a PR for a given release, add `help wanted` to good issues for new community members, and `blocker` for issues that scare you or need immediate attention. Request for more information from a submitter if an issue is not clear. Create new labels as needed by the project.
-
-### Be Responsive
-
-Respond to enhancement requests, and forum posts. Allocate time to reviewing and commenting on issues and conversations as they come in.
-
-### Maintain Overall Health of the Repo
-
-Keep the `main` branch at production quality at all times. Backport features as needed. Cut release branches and tags to enable future patches.
-
-### Use Semver
-
-Use and enforce [semantic versioning](https://semver.org/) and do not let breaking changes be made outside of major releases.
 
-### Release Frequently
+| Maintainer              | GitHub ID                                   | Affiliation |
+| ----------------------- | ------------------------------------------- | ----------- |
+| Asif Sohail Mohammed | [asifsmohammed](https://github.com/asifsmohammed)     | Amazon      |
+| David Venable        | [dlvenable](https://github.com/dlvenable)             | Amazon      |
+| Hai Yan              | [oeyh](https://github.com/oeyh)                       | Amazon      |
 
-Make frequent project releases to the community.
 
-### Promote Other Maintainers
 
-Assist, add, and remove [MAINTAINERS](MAINTAINERS.md). Exercise good judgement, and propose high quality contributors to become co-maintainers.
+## Emeritus
 
+| Maintainer              | GitHub ID                                   | Affiliation |
+| ----------------------- | ------------------------------------------- | ----------- |
+| Jack Mazanec            | [jmazanec15](https://github.com/jmazanec15) | Amazon      |
+| Vamshi Vijay Nakkirtha  | [vamshin](https://github.com/vamshin)       | Amazon      |
+| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB)     | Amazon      |
+| Deep Datta              | [deepdatta](https://github.com/deepdatta)   | Amazon      |
+| Shivani Shukla          | [sshivanii](https://github.com/sshivanii)   | Amazon      |

data/README.md

@@ -18,7 +18,8 @@ The logstash-output-opensearch plugin helps to ship events from Logstash to Open
 ## Project Resources
 
 * [Project Website](https://opensearch.org/)
-* [Documentation](https://opensearch.org/docs/clients/logstash/index/)
+* [Detailed Documentation](https://opensearch.org/docs/latest/tools/logstash/ship-to-opensearch/)
+* [Logstash Overview](https://opensearch.org/docs/clients/logstash/index/)
 * [Developer Guide](DEVELOPER_GUIDE.md)
 * Need help? Try [Forums](https://discuss.opendistrocommunity.dev/)
 * [Project Principles](https://opensearch.org/#principles)
@@ -44,17 +45,17 @@ output {
 
 To run the Logstash Output Opensearch plugin using aws_iam authentication, refer to the sample configuration shown below:
 ```
-output {        
-   opensearch {     
-          hosts => ["hostname:port"]              
-          auth_type => {    
-              type => 'aws_iam'     
-              aws_access_key_id => 'ACCESS_KEY'     
-              aws_secret_access_key => 'SECRET_KEY'     
-              region => 'us-west-2'         
-          }         
-          index  => "logstash-logs-%{+YYYY.MM.dd}"      
-   }            
+output {
+   opensearch {
+          hosts => ["hostname:port"]
+          auth_type => {
+              type => 'aws_iam'
+              aws_access_key_id => 'ACCESS_KEY'
+              aws_secret_access_key => 'SECRET_KEY'
+              region => 'us-west-2'
+          }
+          index  => "logstash-logs-%{+YYYY.MM.dd}"
+   }
 }
 ```
 
@@ -62,37 +63,53 @@ In addition to the existing authentication mechanisms, if we want to add new aut
 
 Example Configuration for basic authentication:
 ```
-output {    
-    opensearch {        
-          hosts  => ["hostname:port"]     
-          auth_type => {            
-              type => 'basic'           
-              user => 'admin'           
-              password => 'admin'           
-          }             
-          index => "logstash-logs-%{+YYYY.MM.dd}"       
-   }            
-}  
+output {
+    opensearch {
+          hosts  => ["hostname:port"]
+          auth_type => {
+              type => 'basic'
+              user => 'admin'
+              password => 'admin'
+          }
+          index => "logstash-logs-%{+YYYY.MM.dd}"
+   }
+}
 ```
 
 To ingest data into a `data stream` through logstash, we need to create the data stream and specify the name of data stream and the `op_type` of `create` in the output configuration. The sample configuration is shown below:
 
 ```yml
-output {    
-    opensearch {        
-          hosts  => ["https://hostname:port"]     
-          auth_type => {            
-              type => 'basic'           
-              user => 'admin'           
-              password => 'admin'           
+output {
+    opensearch {
+          hosts  => ["https://hostname:port"]
+          auth_type => {
+              type => 'basic'
+              user => 'admin'
+              password => 'admin'
           }
           index => "my-data-stream"
           action => "create"
-   }            
-}               
+   }
+}
+```
+
+Starting in 2.0.0, the aws sdk version is bumped to v3. In order for all other AWS plugins to work together, please remove pre-installed aws plugins and install logstash-integration-aws plugin as follows. See also https://github.com/logstash-plugins/logstash-mixin-aws/issues/38
+```
+# Remove existing logstash aws plugins and install logstash-integration-aws to keep sdk dependency the same
+# https://github.com/logstash-plugins/logstash-mixin-aws/issues/38
+/usr/share/logstash/bin/logstash-plugin remove logstash-input-s3
+/usr/share/logstash/bin/logstash-plugin remove logstash-input-sqs
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-s3
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-sns
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-sqs
+/usr/share/logstash/bin/logstash-plugin remove logstash-output-cloudwatch
+
+/usr/share/logstash/bin/logstash-plugin install --version 0.1.0.pre logstash-integration-aws
+bin/logstash-plugin install --version 2.0.0 logstash-output-opensearch
 ```
+## ECS Compatibility
+[Elastic Common Schema(ECS)](https://www.elastic.co/guide/en/ecs/current/index.html]) compatibility for V8 was added in 1.3.0. For more details on ECS support refer to this [documentation](docs/ecs_compatibility.md).
 
-For more details refer to this [documentation](https://opensearch.org/docs/latest/clients/logstash/ship-to-opensearch/#opensearch-output-plugin)
 
 ## Code of Conduct
 
@@ -104,4 +121,4 @@ This project is licensed under the [Apache v2.0 License](LICENSE).
 
 ## Copyright
 
-Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.
+Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.

data/RELEASING.md

@@ -33,4 +33,8 @@ Repositories create consistent release labels, such as `v1.0.0`, `v1.1.0` and `v
 
 The release process is standard across repositories in this org and is run by a release manager volunteering from amongst [MAINTAINERS](MAINTAINERS.md).
 
-TODO
+1. Create a tag, e.g. 1.0.0, and push it to this GitHub repository.
+1. The [release-drafter.yml](.github/workflows/release-drafter.yml) will be automatically kicked off and a draft release will be created.
+1. This draft release triggers the [jenkins release workflow](https://build.ci.opensearch.org/job/logstash-ouput-opensearch-release) as a As a result of which the logstash-output-plugin is released on [rubygems.org](https://rubygems.org/gems/logstash-output-opensearch). Please note that the release workflow is triggered only if created release is in draft state.
+1. Once the above release workflow is successful, the drafted release on GitHub is published automatically.
+1. Increment "version" in [logstash-output-opensearch.gemspec](./logstash-output-opensearch.gemspec) to the next iteration, e.g. 1.0.1.

data/docs/ecs_compatibility.md

@@ -0,0 +1,42 @@
+# ECS Compatibility Support in logstash-output-opensearch
+Compatibility for ECS v8 was added in release 1.3.0 of the plugin. This would enable the plugin to work with Logstash 8.x without the need to disable ecs_compatibility.  
+The output plugin doesn't create any events itself, but merely forwards events to OpenSearch that were shaped by plugins preceding it in the pipeline. So, it doesn't play a direct role in making the events ECS compatible.  
+However, the default index templates that the plugin installs into OpenSearch in the ecs_compatibility modes (v1 & v8) ensures that the document fields stored in the indices are ECS compatible. OpenSearch would throw errors for documents that have fields which are ECS incompatible and can't be coerced. 
+
+## ECS index templates used by logstash-output-opensearch 1.3.0
+* v8 [ECS 8.0.0](https://github.com/elastic/ecs/releases/tag/v8.0.0) [ecs-8.0.0/generated/elasticsearch/legacy/template.json](https://raw.githubusercontent.com/elastic/ecs/v8.0.0/generated/elasticsearch/legacy/template.json)
+* v1 [ECS 1.9.0](https://github.com/elastic/ecs/releases/tag/v1.9.0) [ecs-1.9.0/generated/elasticsearch/7/template.json](https://raw.githubusercontent.com/elastic/ecs/v1.9.0/generated/elasticsearch/7/template.json)
+  
+## ECS incompatibility
+Incompatibility can arise for an event when it has fields with the same name as an ECS defined field but a type which can't be coerced into the ECS field.  
+It is Ok to have fields that are not defined in ECS [ref](https://www.elastic.co/guide/en/ecs/current/ecs-faq.html#addl-fields).
+The dynamic template included in the ECS templates would dynamically map any non-ECS fields. 
+
+### Example 
+[ECS defines the "server"](https://www.elastic.co/guide/en/ecs/current/ecs-server.html) field as an object. But let's say the plugin gets the event below, with a string in the `server` field. It will receive an error from OpenSearch as strings can't be coerced into an object and the ECS incompatible event won't be indexed.
+```
+{
+	"@timestamp": "2022-08-22T15:39:18.142175244Z",
+	"@version": "1",
+	"message": "Doc1",
+	"server": "remoteserver.com"
+}
+```
+
+Error received from OpenSearch in the Logstash logs
+```
+[2022-08-23T00:01:53,366][WARN ][logstash.outputs.opensearch][main][a36555c6fad3f301db8efff2dfbed768fd85e0b6f4ee35626abe62432f83b95d] Could not index event to OpenSearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"ecs-logstash-2022.08.23", :routing=>nil}, {"@timestamp"=>2022-08-22T15:39:18.142175244Z, "@version"=>"1", "server"=>"remoteserver.com", "message"=>"Doc1"}], :response=>{"index"=>{"_index"=>"ecs-logstash-2022.08.23", "_id"=>"CAEUyYIBQM7JQrwxF5NR", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [server] tried to parse field [server] as object, but found a concrete value"}}}}
+```
+## How to ensure ECS compatibility
+* The plugins in the pipeline that create the events like the `input` and `codec` plugins should all use ECS defined fields. 
+* Filter plugins like [mutate](https://github.com/logstash-plugins/logstash-filter-mutate/blob/main/docs/index.asciidoc) can be used to map incompatible fields into ECS compatible ones.  
+In the above example the `server` field can be mapped to the `server.domain` field to make it compatible.  
+* You can use your own custom template in the plugin using the `template` and `template_name` configs.  
+ [According to this](https://www.elastic.co/guide/en/ecs/current/ecs-faq.html#type-interop) some field types can be changed while staying compatible.
+
+As a last resort the `ecs_compatibility` config for the logstash-output-opensearch can be set to `disabled`.
+     
+
+_______________
+## References
+* [Elastic Common Schema (ECS) Reference](https://www.elastic.co/guide/en/ecs/current/index.html)

data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb

@@ -12,6 +12,9 @@ require 'cgi'
 require 'manticore'
 require 'uri'
 
+java_import 'org.apache.http.util.EntityUtils'
+java_import 'org.apache.http.entity.StringEntity'
+
 module LogStash; module Outputs; class OpenSearch; class HttpClient;
   AWS_DEFAULT_PORT = 443
   AWS_DEFAULT_PROFILE = 'default'
@@ -57,7 +60,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       if options[:proxy]
         options[:proxy] = manticore_proxy_hash(options[:proxy])
       end
-      
+
       @manticore = ::Manticore::Client.new(options)
     end
 
@@ -76,6 +79,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       instance_cred_timeout = options[:auth_type]["instance_profile_credentials_timeout"] || AWS_DEFAULT_PROFILE_CREDENTIAL_TIMEOUT
       region = options[:auth_type]["region"] || AWS_DEFAULT_REGION
       set_aws_region(region)
+      set_service_name(options[:auth_type]["service_name"] || AWS_SERVICE)
 
       credential_config = AWSIAMCredential.new(aws_access_key_id, aws_secret_access_key, session_token, profile, instance_cred_retries, instance_cred_timeout, region)
       @credentials = Aws::CredentialProviderChain.new(credential_config).resolve
@@ -93,6 +97,14 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       @region
     end
 
+    def set_service_name(service_name)
+      @service_name = service_name
+    end
+
+    def get_service_name()
+      @service_name
+    end
+
     def set_user_password(options)
       @user = options[:auth_type]["user"]
       @password = options[:auth_type]["password"]
@@ -105,7 +117,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
     def get_password()
       @password
     end
-    
+
     # Transform the proxy option to a hash. Manticore's support for non-hash
     # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
     # but this is not yet released
@@ -137,12 +149,12 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       params[:body] = body if body
 
       if url.user
-        params[:auth] = { 
+        params[:auth] = {
           :user => CGI.unescape(url.user),
           # We have to unescape the password here since manticore won't do it
           # for us unless its part of the URL
-          :password => CGI.unescape(url.password), 
-          :eager => true 
+          :password => CGI.unescape(url.password),
+          :eager => true
         }
       elsif @type == BASIC_AUTH_TYPE
         add_basic_auth_to_params(params)
@@ -172,13 +184,32 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       resp
     end
 
+    # from Manticore, https://github.com/cheald/manticore/blob/acc25cac2999f4658a77a0f39f60ddbca8fe14a4/lib/manticore/client.rb#L536
+    ISO_8859_1 = "ISO-8859-1".freeze
+
+    def minimum_encoding_for(string)
+      if string.ascii_only?
+        ISO_8859_1
+      else
+        string.encoding.to_s
+      end
+    end
+
     def sign_aws_request(request_uri, path, method, params)
       url = URI::HTTPS.build({:host=>URI(request_uri.to_s).host, :port=>AWS_DEFAULT_PORT.to_s, :path=>path})
-      key = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
-                                                         :headers => params[:headers],:body => params[:body]})
-      aws_signer = Aws::Signers::V4.new(@credentials,  AWS_SERVICE, get_aws_region )
-      signed_key =  aws_signer.sign(key)
-      params[:headers] =  params[:headers].merge(signed_key.headers)
+
+      request = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
+        :headers => params[:headers],:body => params[:body]})
+
+      aws_signer = Aws::Sigv4::Signer.new(service: @service_name, region: @region, credentials_provider: @credentials)
+      signed_key = aws_signer.sign_request(
+        http_method: request.http_method,
+        url: url,
+        headers: params[:headers],
+        # match encoding of the HTTP adapter, see https://github.com/opensearch-project/logstash-output-opensearch/issues/207
+        body: params[:body] ? EntityUtils.toString(StringEntity.new(params[:body], minimum_encoding_for(params[:body]))) : nil
+      )
+      params[:headers] = params[:headers].merge(signed_key.headers)
     end
 
     def add_basic_auth_to_params(params)
@@ -192,7 +223,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
     # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone
-      
+
       # We excise auth info from the URL in case manticore itself tries to stick
       # sensitive data in a thrown exception or log data
       request_uri.user = nil
@@ -208,7 +239,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
         part && !part.empty? # Skip empty nil and ""
       end
-      
+
       request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
 
       # use `raw_path`` as `path` will unescape any escaped '/' in the path

data/lib/logstash/outputs/opensearch/http_client/pool.rb

@@ -40,6 +40,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
     end
 
     attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path
+    attr_reader :default_server_major_version
 
     ROOT_URI_PATH = '/'.freeze
 
@@ -68,6 +69,7 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
         @resurrect_delay = merged[:resurrect_delay]
         @sniffing = merged[:sniffing]
         @sniffer_delay = merged[:sniffer_delay]
+        @default_server_major_version = merged[:default_server_major_version]
       end
 
       # Used for all concurrent operations in this class
@@ -412,8 +414,15 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
     end
 
     def get_version(url)
-      request = perform_request_to_url(url, :get, ROOT_URI_PATH)
-      LogStash::Json.load(request.body)["version"]["number"] # e.g. "7.10.0"
+      response = perform_request_to_url(url, :get, ROOT_URI_PATH)
+      if response.code != 404 && !response.body.empty?
+        return LogStash::Json.load(response.body)["version"]["number"] # e.g. "7.10.0"
+      end
+      if @default_server_major_version.nil?
+        @logger.error("Failed to get version from health_check endpoint and default_server_major_version is not configured.")
+        raise "get_version failed! no default_server_major_version configured."
+      end
+      "#{default_server_major_version}.0.0"
     end
 
     def last_version

data/lib/logstash/outputs/opensearch/http_client.rb

@@ -107,7 +107,6 @@ module LogStash; module Outputs; class OpenSearch;
 
       body_stream = StringIO.new
       if http_compression
-        body_stream.set_encoding "BINARY"
         stream_writer = gzip_writer(body_stream)
       else
         stream_writer = body_stream
@@ -126,9 +125,24 @@ module LogStash; module Outputs; class OpenSearch;
                        :payload_size => stream_writer.pos,
                        :content_length => body_stream.size,
                        :batch_offset => (index + 1 - batch_actions.size))
+
+          # Have to close gzip writer before reading from body_stream; otherwise stream doesn't end properly
+          # and will cause server side error
+          if http_compression
+            stream_writer.close
+          end
+
           bulk_responses << bulk_send(body_stream, batch_actions)
-          body_stream.truncate(0) && body_stream.seek(0)
-          stream_writer = gzip_writer(body_stream) if http_compression
+
+          if http_compression
+            # Get a new StringIO object and gzip writer
+            body_stream = StringIO.new
+            stream_writer = gzip_writer(body_stream)
+          else
+            # Clear existing StringIO object and reuse existing stream writer
+            body_stream.truncate(0) && body_stream.seek(0)
+          end
+
           batch_actions.clear
         end
         stream_writer.write(as_json)
@@ -149,6 +163,7 @@ module LogStash; module Outputs; class OpenSearch;
       fail(ArgumentError, "Cannot create gzip writer on IO with unread bytes") unless io.eof?
       fail(ArgumentError, "Cannot create gzip writer on non-empty IO") unless io.pos == 0
 
+      io.set_encoding "BINARY"
       Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
     end
 
@@ -328,7 +343,8 @@ module LogStash; module Outputs; class OpenSearch;
         :healthcheck_path => options[:healthcheck_path],
         :resurrect_delay => options[:resurrect_delay],
         :url_normalizer => self.method(:host_to_url),
-        :metric => options[:metric]
+        :metric => options[:metric],
+        :default_server_major_version => options[:default_server_major_version]
       }
       pool_options[:scheme] = self.scheme if self.scheme
 
@@ -388,10 +404,16 @@ module LogStash; module Outputs; class OpenSearch;
       @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
+    def legacy_template?()
+      # TODO: Also check Version and return true for < 7.8 even if :legacy_template=false
+      # Need to figure a way to distinguish between OpenSearch, OpenDistro and other 
+      # variants, since they have version numbers in different ranges.
+      client_settings.fetch(:legacy_template, true)
+    end
+
     def template_endpoint
-      # TODO: Check Version < 7.8 and use index template for >= 7.8 & OpenSearch
       # https://opensearch.org/docs/opensearch/index-templates/
-      '_template'
+      legacy_template?() ? '_template' : '_index_template'
     end
 
     # check whether rollover alias already exists

data/lib/logstash/outputs/opensearch/http_client_builder.rb

@@ -18,7 +18,8 @@ module LogStash; module Outputs; class OpenSearch;
         :pool_max_per_route => params["pool_max_per_route"],
         :check_connection_timeout => params["validate_after_inactivity"],
         :http_compression => params["http_compression"],
-        :headers => params["custom_headers"] || {}
+        :headers => params["custom_headers"] || {},
+        :legacy_template => params["legacy_template"]
       }
       
       client_settings[:proxy] = params["proxy"] if params["proxy"]
@@ -26,7 +27,8 @@ module LogStash; module Outputs; class OpenSearch;
       common_options = {
         :client_settings => client_settings,
         :metric => params["metric"],
-        :resurrect_delay => params["resurrect_delay"]
+        :resurrect_delay => params["resurrect_delay"],
+        :default_server_major_version => params["default_server_major_version"]
       }
 
       if params["sniffing"]

data/lib/logstash/outputs/opensearch/template_manager.rb

@@ -18,7 +18,7 @@ module LogStash; module Outputs; class OpenSearch
       else
         plugin.logger.info("Using a default mapping template", :version => plugin.maximum_seen_major_version,
                                                                :ecs_compatibility => plugin.ecs_compatibility)
-        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
+        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility, plugin.client.legacy_template?)
       end
 
       plugin.logger.debug("Attempting to install template", template: template)
@@ -26,8 +26,8 @@ module LogStash; module Outputs; class OpenSearch
     end
 
     private
-    def self.load_default_template(major_version, ecs_compatibility)
-      template_path = default_template_path(major_version, ecs_compatibility)
+    def self.load_default_template(major_version, ecs_compatibility, legacy_template)
+      template_path = default_template_path(major_version, ecs_compatibility, legacy_template)
       read_template_file(template_path)
     rescue => e
       fail "Failed to load default template for OpenSearch v#{major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
@@ -45,9 +45,10 @@ module LogStash; module Outputs; class OpenSearch
       plugin.template_name
     end
 
-    def self.default_template_path(major_version, ecs_compatibility=:disabled)
+    def self.default_template_path(major_version, ecs_compatibility=:disabled, legacy_template=true)
       template_version = major_version
-      default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x.json"
+      suffix = legacy_template ? "" : "_index"
+      default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x#{suffix}.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end
 

data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json

@@ -0,0 +1,66 @@
+{
+  "index_patterns": "logstash-*",
+  "version": 60001,
+  "priority": 10,
+  "template": {
+    "settings": {
+      "index.refresh_interval": "5s",
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "path_match": "message",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false
+            }
+          }
+        },
+        {
+          "string_fields": {
+            "match": "*",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false,
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            }
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "geoip": {
+          "dynamic": true,
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "latitude": {
+              "type": "half_float"
+            },
+            "longitude": {
+              "type": "half_float"
+            }
+          }
+        }
+      }
+    }
+  }
+}