logstash-output-elasticsearch-test 10.3.0-x86_64-linux

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json

@@ -0,0 +1,95 @@
+{
+  "template" : "logstash-*",
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "_all" : {"enabled" : true, "omit_norms" : true},
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" }
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" },
+            "fields" : {
+              "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
+            }
+          }
+        }
+      }, {
+        "float_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "float",
+          "mapping" : { "type" : "float", "doc_values" : true }
+        }
+      }, {
+        "double_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "double",
+          "mapping" : { "type" : "double", "doc_values" : true }
+        }
+      }, {
+        "byte_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "byte",
+          "mapping" : { "type" : "byte", "doc_values" : true }
+        }
+      }, {
+        "short_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "short",
+          "mapping" : { "type" : "short", "doc_values" : true }
+        }
+      }, {
+        "integer_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "integer",
+          "mapping" : { "type" : "integer", "doc_values" : true }
+        }
+      }, {
+        "long_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "long",
+          "mapping" : { "type" : "long", "doc_values" : true }
+        }
+      }, {
+        "date_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "date",
+          "mapping" : { "type" : "date", "doc_values" : true }
+        }
+      }, {
+        "geo_point_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "geo_point",
+          "mapping" : { "type" : "geo_point", "doc_values" : true }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date", "doc_values" : true },
+        "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
+        "geoip"  : {
+          "type" : "object",
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip", "doc_values" : true },
+            "location" : { "type" : "geo_point", "doc_values" : true },
+            "latitude" : { "type" : "float", "doc_values" : true },
+            "longitude" : { "type" : "float", "doc_values" : true }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json

@@ -0,0 +1,46 @@
+{
+  "template" : "logstash-*",
+  "version" : 50001,
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "_all" : {"enabled" : true, "norms" : false},
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date", "include_in_all": false },
+        "@version": { "type": "keyword", "include_in_all": false },
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json

@@ -0,0 +1,45 @@
+{
+  "template" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date"},
+        "@version": { "type": "keyword"},
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json

@@ -0,0 +1,44 @@
+{
+  "index_patterns" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s",
+    "number_of_shards": 1
+  },
+  "mappings" : {
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
+        }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
+          }
+        }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json

@@ -0,0 +1,44 @@
+{
+  "index_patterns" : "logstash-*",
+  "version" : 80001,
+  "settings" : {
+    "index.refresh_interval" : "5s",
+    "number_of_shards": 1
+  },
+  "mappings" : {
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
+        }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
+          }
+        }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -0,0 +1,131 @@
+require 'manticore'
+require 'cgi'
+
+module LogStash; module Outputs; class ElasticSearch; class HttpClient;
+  DEFAULT_HEADERS = { "Content-Type" => "application/json" }
+  
+  class ManticoreAdapter
+    attr_reader :manticore, :logger
+
+    def initialize(logger, options={})
+      @logger = logger
+      options = options.clone || {}
+      options[:ssl] = options[:ssl] || {}
+
+      # We manage our own retries directly, so let's disable them here
+      options[:automatic_retries] = 0
+      # We definitely don't need cookies
+      options[:cookies] = false
+
+      @client_params = {:headers => DEFAULT_HEADERS.merge(options[:headers] || {})}
+      
+      if options[:proxy]
+        options[:proxy] = manticore_proxy_hash(options[:proxy])
+      end
+      
+      @manticore = ::Manticore::Client.new(options)
+    end
+    
+    # Transform the proxy option to a hash. Manticore's support for non-hash
+    # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
+    # but this is not yet released
+    def manticore_proxy_hash(proxy_uri)
+      [:scheme, :port, :user, :password, :path].reduce(:host => proxy_uri.host) do |acc,opt|
+        value = proxy_uri.send(opt)
+        acc[opt] = value unless value.nil? || (value.is_a?(String) && value.empty?)
+        acc
+      end
+    end
+
+    def client
+      @manticore
+    end
+
+    # Performs the request by invoking {Transport::Base#perform_request} with a block.
+    #
+    # @return [Response]
+    # @see    Transport::Base#perform_request
+    #
+    def perform_request(url, method, path, params={}, body=nil)
+      # Perform 2-level deep merge on the params, so if the passed params and client params will both have hashes stored on a key they
+      # will be merged as well, instead of choosing just one of the values
+      params = (params || {}).merge(@client_params) { |key, oldval, newval|
+        (oldval.is_a?(Hash) && newval.is_a?(Hash)) ? oldval.merge(newval) : newval
+      }
+      params[:body] = body if body
+
+      if url.user
+        params[:auth] = { 
+          :user => CGI.unescape(url.user),
+          # We have to unescape the password here since manticore won't do it
+          # for us unless its part of the URL
+          :password => CGI.unescape(url.password), 
+          :eager => true 
+        }
+      end
+
+      request_uri = format_url(url, path)
+      request_uri_as_string = remove_double_escaping(request_uri.to_s)
+      resp = @manticore.send(method.downcase, request_uri_as_string, params)
+
+      # Manticore returns lazy responses by default
+      # We want to block for our usage, this will wait for the repsonse
+      # to finish
+      resp.call
+
+      # 404s are excluded because they are valid codes in the case of
+      # template installation. We might need a better story around this later
+      # but for our current purposes this is correct
+      if resp.code < 200 || resp.code > 299 && resp.code != 404
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body)
+      end
+
+      resp
+    end
+
+    # Returned urls from this method should be checked for double escaping.
+    def format_url(url, path_and_query=nil)
+      request_uri = url.clone
+      
+      # We excise auth info from the URL in case manticore itself tries to stick
+      # sensitive data in a thrown exception or log data
+      request_uri.user = nil
+      request_uri.password = nil
+
+      return request_uri.to_s if path_and_query.nil?
+
+      parsed_path_and_query = java.net.URI.new(path_and_query)
+
+      query = request_uri.query
+      parsed_query = parsed_path_and_query.query
+
+      new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
+        part && !part.empty? # Skip empty nil and ""
+      end
+      
+      request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
+
+      # use `raw_path`` as `path` will unescape any escaped '/' in the path
+      request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.raw_path}".gsub(/\/{2,}/, "/")
+      request_uri
+    end
+
+    # Later versions of SafeURI will also escape the '%' sign in an already escaped URI.
+    # (If the path variable is used, it constructs a new java.net.URI object using the multi-arg constructor,
+    # which will escape any '%' characters in the path, as opposed to the single-arg constructor which requires illegal
+    # characters to be already escaped, and will throw otherwise)
+    # The URI needs to have been previously escaped, as it does not play nice with an escaped '/' in the
+    # middle of a URI, as required by date math, treating it as a path separator
+    def remove_double_escaping(url)
+      url.gsub(/%25([0-9A-F]{2})/i, '%\1')
+    end
+
+    def close
+      @manticore.close
+    end
+
+    def host_unreachable_exceptions
+      [::Manticore::Timeout,::Manticore::SocketException, ::Manticore::ClientProtocolException, ::Manticore::ResolutionFailure, Manticore::SocketTimeout]
+    end
+  end
+end; end; end; end