logstash-output-opensearch 1.3.0-java → 2.0.3-java

data/lib/logstash/outputs/opensearch/http_client.rb

@@ -107,7 +107,6 @@ module LogStash; module Outputs; class OpenSearch;
 
       body_stream = StringIO.new
       if http_compression
-        body_stream.set_encoding "BINARY"
         stream_writer = gzip_writer(body_stream)
       else
         stream_writer = body_stream
@@ -126,9 +125,24 @@ module LogStash; module Outputs; class OpenSearch;
                        :payload_size => stream_writer.pos,
                        :content_length => body_stream.size,
                        :batch_offset => (index + 1 - batch_actions.size))
+
+          # Have to close gzip writer before reading from body_stream; otherwise stream doesn't end properly
+          # and will cause server side error
+          if http_compression
+            stream_writer.close
+          end
+
           bulk_responses << bulk_send(body_stream, batch_actions)
-          body_stream.truncate(0) && body_stream.seek(0)
-          stream_writer = gzip_writer(body_stream) if http_compression
+
+          if http_compression
+            # Get a new StringIO object and gzip writer
+            body_stream = StringIO.new
+            stream_writer = gzip_writer(body_stream)
+          else
+            # Clear existing StringIO object and reuse existing stream writer
+            body_stream.truncate(0) && body_stream.seek(0)
+          end
+
           batch_actions.clear
         end
         stream_writer.write(as_json)
@@ -149,6 +163,7 @@ module LogStash; module Outputs; class OpenSearch;
       fail(ArgumentError, "Cannot create gzip writer on IO with unread bytes") unless io.eof?
       fail(ArgumentError, "Cannot create gzip writer on non-empty IO") unless io.pos == 0
 
+      io.set_encoding "BINARY"
       Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
     end
 
@@ -328,7 +343,8 @@ module LogStash; module Outputs; class OpenSearch;
         :healthcheck_path => options[:healthcheck_path],
         :resurrect_delay => options[:resurrect_delay],
         :url_normalizer => self.method(:host_to_url),
-        :metric => options[:metric]
+        :metric => options[:metric],
+        :default_server_major_version => options[:default_server_major_version]
       }
       pool_options[:scheme] = self.scheme if self.scheme
 
@@ -388,10 +404,16 @@ module LogStash; module Outputs; class OpenSearch;
       @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
+    def legacy_template?()
+      # TODO: Also check Version and return true for < 7.8 even if :legacy_template=false
+      # Need to figure a way to distinguish between OpenSearch, OpenDistro and other 
+      # variants, since they have version numbers in different ranges.
+      client_settings.fetch(:legacy_template, true)
+    end
+
     def template_endpoint
-      # TODO: Check Version < 7.8 and use index template for >= 7.8 & OpenSearch
       # https://opensearch.org/docs/opensearch/index-templates/
-      '_template'
+      legacy_template?() ? '_template' : '_index_template'
     end
 
     # check whether rollover alias already exists

data/lib/logstash/outputs/opensearch/http_client_builder.rb

@@ -18,7 +18,8 @@ module LogStash; module Outputs; class OpenSearch;
         :pool_max_per_route => params["pool_max_per_route"],
         :check_connection_timeout => params["validate_after_inactivity"],
         :http_compression => params["http_compression"],
-        :headers => params["custom_headers"] || {}
+        :headers => params["custom_headers"] || {},
+        :legacy_template => params["legacy_template"]
       }
       
       client_settings[:proxy] = params["proxy"] if params["proxy"]
@@ -26,7 +27,8 @@ module LogStash; module Outputs; class OpenSearch;
       common_options = {
         :client_settings => client_settings,
         :metric => params["metric"],
-        :resurrect_delay => params["resurrect_delay"]
+        :resurrect_delay => params["resurrect_delay"],
+        :default_server_major_version => params["default_server_major_version"]
       }
 
       if params["sniffing"]

data/lib/logstash/outputs/opensearch/template_manager.rb

@@ -18,7 +18,7 @@ module LogStash; module Outputs; class OpenSearch
       else
         plugin.logger.info("Using a default mapping template", :version => plugin.maximum_seen_major_version,
                                                                :ecs_compatibility => plugin.ecs_compatibility)
-        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
+        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility, plugin.client.legacy_template?)
       end
 
       plugin.logger.debug("Attempting to install template", template: template)
@@ -26,8 +26,8 @@ module LogStash; module Outputs; class OpenSearch
     end
 
     private
-    def self.load_default_template(major_version, ecs_compatibility)
-      template_path = default_template_path(major_version, ecs_compatibility)
+    def self.load_default_template(major_version, ecs_compatibility, legacy_template)
+      template_path = default_template_path(major_version, ecs_compatibility, legacy_template)
       read_template_file(template_path)
     rescue => e
       fail "Failed to load default template for OpenSearch v#{major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
@@ -45,9 +45,10 @@ module LogStash; module Outputs; class OpenSearch
       plugin.template_name
     end
 
-    def self.default_template_path(major_version, ecs_compatibility=:disabled)
+    def self.default_template_path(major_version, ecs_compatibility=:disabled, legacy_template=true)
       template_version = major_version
-      default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x.json"
+      suffix = legacy_template ? "" : "_index"
+      default_template_name = "templates/ecs-#{ecs_compatibility}/#{template_version}x#{suffix}.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end
 

data/lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json

@@ -0,0 +1,66 @@
+{
+  "index_patterns": "logstash-*",
+  "version": 60001,
+  "priority": 10,
+  "template": {
+    "settings": {
+      "index.refresh_interval": "5s",
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "path_match": "message",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false
+            }
+          }
+        },
+        {
+          "string_fields": {
+            "match": "*",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false,
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            }
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "geoip": {
+          "dynamic": true,
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "latitude": {
+              "type": "half_float"
+            },
+            "longitude": {
+              "type": "half_float"
+            }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json

@@ -0,0 +1,66 @@
+{
+  "index_patterns": "logstash-*",
+  "version": 60001,
+  "priority": 10,
+  "template": {
+    "settings": {
+      "index.refresh_interval": "5s",
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "path_match": "message",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false
+            }
+          }
+        },
+        {
+          "string_fields": {
+            "match": "*",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false,
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            }
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "geoip": {
+          "dynamic": true,
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "latitude": {
+              "type": "half_float"
+            },
+            "longitude": {
+              "type": "half_float"
+            }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json

@@ -0,0 +1,66 @@
+{
+  "index_patterns": "logstash-*",
+  "version": 60001,
+  "priority": 10,
+  "template": {
+    "settings": {
+      "index.refresh_interval": "5s",
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "path_match": "message",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false
+            }
+          }
+        },
+        {
+          "string_fields": {
+            "match": "*",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false,
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            }
+          }
+        }
+      ],
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "geoip": {
+          "dynamic": true,
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "latitude": {
+              "type": "half_float"
+            },
+            "longitude": {
+              "type": "half_float"
+            }
+          }
+        }
+      }
+    }
+  }
+}