logstash-output-elasticsearch-test 10.3.0-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +397 -0
- data/CONTRIBUTORS +33 -0
- data/Gemfile +15 -0
- data/LICENSE +13 -0
- data/NOTICE.TXT +5 -0
- data/README.md +106 -0
- data/docs/index.asciidoc +899 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +441 -0
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +167 -0
- data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json +95 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json +46 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json +45 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json +44 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json +44 -0
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +131 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +495 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +432 -0
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +159 -0
- data/lib/logstash/outputs/elasticsearch/ilm.rb +113 -0
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +61 -0
- data/lib/logstash/outputs/elasticsearch.rb +263 -0
- data/logstash-output-elasticsearch.gemspec +33 -0
- data/spec/es_spec_helper.rb +189 -0
- data/spec/fixtures/_nodes/2x_1x.json +27 -0
- data/spec/fixtures/_nodes/5x_6x.json +81 -0
- data/spec/fixtures/_nodes/7x.json +92 -0
- data/spec/fixtures/htpasswd +2 -0
- data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
- data/spec/fixtures/scripts/groovy/scripted_update.groovy +2 -0
- data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy +2 -0
- data/spec/fixtures/scripts/groovy/scripted_upsert.groovy +2 -0
- data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
- data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
- data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
- data/spec/fixtures/template-with-policy-es6x.json +48 -0
- data/spec/fixtures/template-with-policy-es7x.json +45 -0
- data/spec/fixtures/test_certs/ca/ca.crt +32 -0
- data/spec/fixtures/test_certs/ca/ca.key +51 -0
- data/spec/fixtures/test_certs/test.crt +36 -0
- data/spec/fixtures/test_certs/test.key +51 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +69 -0
- data/spec/integration/outputs/create_spec.rb +67 -0
- data/spec/integration/outputs/delete_spec.rb +65 -0
- data/spec/integration/outputs/groovy_update_spec.rb +150 -0
- data/spec/integration/outputs/ilm_spec.rb +531 -0
- data/spec/integration/outputs/index_spec.rb +178 -0
- data/spec/integration/outputs/index_version_spec.rb +102 -0
- data/spec/integration/outputs/ingest_pipeline_spec.rb +74 -0
- data/spec/integration/outputs/metrics_spec.rb +70 -0
- data/spec/integration/outputs/no_es_on_startup_spec.rb +58 -0
- data/spec/integration/outputs/painless_update_spec.rb +189 -0
- data/spec/integration/outputs/parent_spec.rb +102 -0
- data/spec/integration/outputs/retry_spec.rb +169 -0
- data/spec/integration/outputs/routing_spec.rb +61 -0
- data/spec/integration/outputs/sniffer_spec.rb +133 -0
- data/spec/integration/outputs/templates_5x_spec.rb +98 -0
- data/spec/integration/outputs/templates_spec.rb +98 -0
- data/spec/integration/outputs/update_spec.rb +116 -0
- data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
- data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
- data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
- data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
- data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
- data/spec/unit/http_client_builder_spec.rb +185 -0
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +149 -0
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +274 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +250 -0
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +25 -0
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +72 -0
- data/spec/unit/outputs/elasticsearch_spec.rb +675 -0
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +82 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +54 -0
- metadata +300 -0
@@ -0,0 +1,61 @@
|
|
1
|
+
module LogStash; module Outputs; class ElasticSearch
|
2
|
+
class TemplateManager
|
3
|
+
# To be mixed into the elasticsearch plugin base
|
4
|
+
def self.install_template(plugin)
|
5
|
+
return unless plugin.manage_template
|
6
|
+
if plugin.template.nil?
|
7
|
+
plugin.logger.info("Using default mapping template")
|
8
|
+
else
|
9
|
+
plugin.logger.info("Using mapping template from", :path => plugin.template)
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
template = get_template(plugin.template, plugin.maximum_seen_major_version)
|
14
|
+
add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
|
15
|
+
plugin.logger.info("Attempting to install template", :manage_template => template)
|
16
|
+
install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
|
17
|
+
rescue => e
|
18
|
+
plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
def self.get_template(path, es_major_version)
|
23
|
+
template_path = path || default_template_path(es_major_version)
|
24
|
+
read_template_file(template_path)
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.install(client, template_name, template, template_overwrite)
|
28
|
+
client.template_install(template_name, template, template_overwrite)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.add_ilm_settings_to_template(plugin, template)
|
32
|
+
# Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern
|
33
|
+
# definition - remove any existing definition of 'template'
|
34
|
+
template.delete('template') if template.include?('template')
|
35
|
+
template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
|
36
|
+
if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
|
37
|
+
plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.")
|
38
|
+
end
|
39
|
+
template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
|
40
|
+
end
|
41
|
+
|
42
|
+
# Template name - if template_name set, use it
|
43
|
+
# if not and ILM is enabled, use the rollover alias
|
44
|
+
# else use the default value of template_name
|
45
|
+
def self.template_name(plugin)
|
46
|
+
plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.default_template_path(es_major_version)
|
50
|
+
template_version = es_major_version == 1 ? 2 : es_major_version
|
51
|
+
default_template_name = "elasticsearch-template-es#{template_version}x.json"
|
52
|
+
::File.expand_path(default_template_name, ::File.dirname(__FILE__))
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.read_template_file(template_path)
|
56
|
+
raise ArgumentError, "Template file '#{template_path}' could not be found!" unless ::File.exists?(template_path)
|
57
|
+
template_data = ::IO.read(template_path)
|
58
|
+
LogStash::Json.load(template_data)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end end end
|
@@ -0,0 +1,263 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/namespace"
|
3
|
+
require "logstash/environment"
|
4
|
+
require "logstash/outputs/base"
|
5
|
+
require "logstash/json"
|
6
|
+
require "concurrent"
|
7
|
+
require "stud/buffer"
|
8
|
+
require "socket" # for Socket.gethostname
|
9
|
+
require "thread" # for safe queueing
|
10
|
+
require "uri" # for escaping user input
|
11
|
+
require "forwardable"
|
12
|
+
|
13
|
+
# .Compatibility Note
|
14
|
+
# [NOTE]
|
15
|
+
# ================================================================================
|
16
|
+
# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
|
17
|
+
# called `http.content_type.required`. If this option is set to `true`, and you
|
18
|
+
# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
|
19
|
+
# plugin to version 6.2.5 or higher.
|
20
|
+
#
|
21
|
+
# ================================================================================
|
22
|
+
#
|
23
|
+
# This plugin is the recommended method of storing logs in Elasticsearch.
|
24
|
+
# If you plan on using the Kibana web interface, you'll want to use this output.
|
25
|
+
#
|
26
|
+
# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
|
27
|
+
# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
|
28
|
+
# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
|
29
|
+
# to upgrade Logstash in lock-step.
|
30
|
+
#
|
31
|
+
# You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
|
32
|
+
#
|
33
|
+
# ==== Template management for Elasticsearch 5.x
|
34
|
+
# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
|
35
|
+
# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
|
36
|
+
# behavior.
|
37
|
+
#
|
38
|
+
# ** Users installing ES 5.x and LS 5.x **
|
39
|
+
# This change will not affect you and you will continue to use the ES defaults.
|
40
|
+
#
|
41
|
+
# ** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
|
42
|
+
# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
|
43
|
+
# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
|
44
|
+
# the new template is installed.
|
45
|
+
#
|
46
|
+
# ==== Retry Policy
|
47
|
+
#
|
48
|
+
# The retry policy has changed significantly in the 2.2.0 release.
|
49
|
+
# This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
|
50
|
+
# either partial or total failures.
|
51
|
+
#
|
52
|
+
# The following errors are retried infinitely:
|
53
|
+
#
|
54
|
+
# - Network errors (inability to connect)
|
55
|
+
# - 429 (Too many requests) and
|
56
|
+
# - 503 (Service unavailable) errors
|
57
|
+
#
|
58
|
+
# NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
|
59
|
+
# It is more performant for Elasticsearch to retry these exceptions than this plugin.
|
60
|
+
#
|
61
|
+
# ==== Batch Sizes ====
|
62
|
+
# This plugin attempts to send batches of events as a single request. However, if
|
63
|
+
# a request exceeds 20MB we will break it up until multiple batch requests. If a single document exceeds 20MB it will be sent as a single request.
|
64
|
+
#
|
65
|
+
# ==== DNS Caching
|
66
|
+
#
|
67
|
+
# This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
|
68
|
+
# a global setting for the JVM.
|
69
|
+
#
|
70
|
+
# As an example, to set your DNS TTL to 1 second you would set
|
71
|
+
# the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
|
72
|
+
#
|
73
|
+
# Keep in mind that a connection with keepalive enabled will
|
74
|
+
# not reevaluate its DNS value while the keepalive is in effect.
|
75
|
+
#
|
76
|
+
# ==== HTTP Compression
|
77
|
+
#
|
78
|
+
# This plugin supports request and response compression. Response compression is enabled by default and
|
79
|
+
# for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for
|
80
|
+
# it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` in
|
81
|
+
# Elasticsearch[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http] to take advantage of response compression when using this plugin
|
82
|
+
#
|
83
|
+
# For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression`
|
84
|
+
# setting in their Logstash config file.
|
85
|
+
#
|
86
|
+
class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
87
|
+
declare_threadsafe!
|
88
|
+
|
89
|
+
require "logstash/outputs/elasticsearch/http_client"
|
90
|
+
require "logstash/outputs/elasticsearch/http_client_builder"
|
91
|
+
require "logstash/outputs/elasticsearch/common_configs"
|
92
|
+
require "logstash/outputs/elasticsearch/common"
|
93
|
+
require "logstash/outputs/elasticsearch/ilm"
|
94
|
+
|
95
|
+
# Protocol agnostic (i.e. non-http, non-java specific) configs go here
|
96
|
+
include(LogStash::Outputs::ElasticSearch::CommonConfigs)
|
97
|
+
|
98
|
+
# Protocol agnostic methods
|
99
|
+
include(LogStash::Outputs::ElasticSearch::Common)
|
100
|
+
|
101
|
+
# Methods for ILM support
|
102
|
+
include(LogStash::Outputs::ElasticSearch::Ilm)
|
103
|
+
|
104
|
+
config_name "elasticsearch"
|
105
|
+
|
106
|
+
# The Elasticsearch action to perform. Valid actions are:
|
107
|
+
#
|
108
|
+
# - index: indexes a document (an event from Logstash).
|
109
|
+
# - delete: deletes a document by id (An id is required for this action)
|
110
|
+
# - create: indexes a document, fails if a document by that id already exists in the index.
|
111
|
+
# - update: updates a document by id. Update has a special case where you can upsert -- update a
|
112
|
+
# document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
|
113
|
+
# in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
|
114
|
+
# - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
|
115
|
+
# would use the foo field for the action
|
116
|
+
#
|
117
|
+
# For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
|
118
|
+
config :action, :validate => :string, :default => "index"
|
119
|
+
|
120
|
+
# Username to authenticate to a secure Elasticsearch cluster
|
121
|
+
config :user, :validate => :string
|
122
|
+
# Password to authenticate to a secure Elasticsearch cluster
|
123
|
+
config :password, :validate => :password
|
124
|
+
|
125
|
+
# Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
|
126
|
+
#
|
127
|
+
# For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
|
128
|
+
config :cloud_auth, :validate => :password
|
129
|
+
|
130
|
+
# HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
|
131
|
+
# the root path for the Elasticsearch HTTP API lives.
|
132
|
+
# Note that if you use paths as components of URLs in the 'hosts' field you may
|
133
|
+
# not also set this field. That will raise an error at startup
|
134
|
+
config :path, :validate => :string
|
135
|
+
|
136
|
+
# HTTP Path to perform the _bulk requests to
|
137
|
+
# this defaults to a concatenation of the path parameter and "_bulk"
|
138
|
+
config :bulk_path, :validate => :string
|
139
|
+
|
140
|
+
# Pass a set of key value pairs as the URL query string. This query string is added
|
141
|
+
# to every host listed in the 'hosts' configuration. If the 'hosts' list contains
|
142
|
+
# urls that already have query strings, the one specified here will be appended.
|
143
|
+
config :parameters, :validate => :hash
|
144
|
+
|
145
|
+
# Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
|
146
|
+
# is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
|
147
|
+
# If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
|
148
|
+
config :ssl, :validate => :boolean
|
149
|
+
|
150
|
+
# Option to validate the server's certificate. Disabling this severely compromises security.
|
151
|
+
# For more information on disabling certificate verification please read
|
152
|
+
# https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
|
153
|
+
config :ssl_certificate_verification, :validate => :boolean, :default => true
|
154
|
+
|
155
|
+
# The .cer or .pem file to validate the server's certificate
|
156
|
+
config :cacert, :validate => :path
|
157
|
+
|
158
|
+
# The JKS truststore to validate the server's certificate.
|
159
|
+
# Use either `:truststore` or `:cacert`
|
160
|
+
config :truststore, :validate => :path
|
161
|
+
|
162
|
+
# Set the truststore password
|
163
|
+
config :truststore_password, :validate => :password
|
164
|
+
|
165
|
+
# The keystore used to present a certificate to the server.
|
166
|
+
# It can be either .jks or .p12
|
167
|
+
config :keystore, :validate => :path
|
168
|
+
|
169
|
+
# Set the keystore password
|
170
|
+
config :keystore_password, :validate => :password
|
171
|
+
|
172
|
+
# This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
|
173
|
+
# Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
|
174
|
+
# this with master nodes, you probably want to disable HTTP on them by setting
|
175
|
+
# `http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
|
176
|
+
# manually enter multiple Elasticsearch hosts using the `hosts` parameter.
|
177
|
+
config :sniffing, :validate => :boolean, :default => false
|
178
|
+
|
179
|
+
# How long to wait, in seconds, between sniffing attempts
|
180
|
+
config :sniffing_delay, :validate => :number, :default => 5
|
181
|
+
|
182
|
+
# HTTP Path to be used for the sniffing requests
|
183
|
+
# the default value is computed by concatenating the path value and "_nodes/http"
|
184
|
+
# if sniffing_path is set it will be used as an absolute path
|
185
|
+
# do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
|
186
|
+
config :sniffing_path, :validate => :string
|
187
|
+
|
188
|
+
# Set the address of a forward HTTP proxy.
|
189
|
+
# This used to accept hashes as arguments but now only accepts
|
190
|
+
# arguments of the URI type to prevent leaking credentials.
|
191
|
+
config :proxy, :validate => :uri
|
192
|
+
|
193
|
+
# Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
|
194
|
+
# a timeout occurs, the request will be retried.
|
195
|
+
config :timeout, :validate => :number, :default => 60
|
196
|
+
|
197
|
+
# Set the Elasticsearch errors in the whitelist that you don't want to log.
|
198
|
+
# A useful example is when you want to skip all 409 errors
|
199
|
+
# which are `document_already_exists_exception`.
|
200
|
+
config :failure_type_logging_whitelist, :validate => :array, :default => []
|
201
|
+
|
202
|
+
# While the output tries to reuse connections efficiently we have a maximum.
|
203
|
+
# This sets the maximum number of open connections the output will create.
|
204
|
+
# Setting this too low may mean frequently closing / opening connections
|
205
|
+
# which is bad.
|
206
|
+
config :pool_max, :validate => :number, :default => 1000
|
207
|
+
|
208
|
+
# While the output tries to reuse connections efficiently we have a maximum per endpoint.
|
209
|
+
# This sets the maximum number of open connections per endpoint the output will create.
|
210
|
+
# Setting this too low may mean frequently closing / opening connections
|
211
|
+
# which is bad.
|
212
|
+
config :pool_max_per_route, :validate => :number, :default => 100
|
213
|
+
|
214
|
+
# HTTP Path where a HEAD request is sent when a backend is marked down
|
215
|
+
# the request is sent in the background to see if it has come back again
|
216
|
+
# before it is once again eligible to service requests.
|
217
|
+
# If you have custom firewall rules you may need to change this
|
218
|
+
config :healthcheck_path, :validate => :string
|
219
|
+
|
220
|
+
# How frequently, in seconds, to wait between resurrection attempts.
|
221
|
+
# Resurrection is the process by which backend endpoints marked 'down' are checked
|
222
|
+
# to see if they have come back to life
|
223
|
+
config :resurrect_delay, :validate => :number, :default => 5
|
224
|
+
|
225
|
+
# How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
|
226
|
+
# You may want to set this lower, if you get connection errors regularly
|
227
|
+
# Quoting the Apache commons docs (this client is based Apache Commmons):
|
228
|
+
# 'Defines period of inactivity in milliseconds after which persistent connections must
|
229
|
+
# be re-validated prior to being leased to the consumer. Non-positive value passed to
|
230
|
+
# this method disables connection validation. This check helps detect connections that
|
231
|
+
# have become stale (half-closed) while kept inactive in the pool.'
|
232
|
+
# See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
|
233
|
+
config :validate_after_inactivity, :validate => :number, :default => 10000
|
234
|
+
|
235
|
+
# Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
|
236
|
+
config :http_compression, :validate => :boolean, :default => false
|
237
|
+
|
238
|
+
# Custom Headers to send on each request to elasticsearch nodes
|
239
|
+
config :custom_headers, :validate => :hash, :default => {}
|
240
|
+
|
241
|
+
def build_client
|
242
|
+
params["metric"] = metric
|
243
|
+
@client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
|
244
|
+
end
|
245
|
+
|
246
|
+
def close
|
247
|
+
@stopping.make_true
|
248
|
+
stop_template_installer
|
249
|
+
@client.close if @client
|
250
|
+
end
|
251
|
+
|
252
|
+
def self.oss?
|
253
|
+
LogStash::OSS
|
254
|
+
end
|
255
|
+
|
256
|
+
@@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
|
257
|
+
|
258
|
+
@@plugins.each do |plugin|
|
259
|
+
name = plugin.name.split('-')[-1]
|
260
|
+
require "logstash/outputs/elasticsearch/#{name}"
|
261
|
+
end
|
262
|
+
|
263
|
+
end # class LogStash::Outputs::Elasticsearch
|
@@ -0,0 +1,33 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-elasticsearch-test'
|
3
|
+
s.version = '10.3.0'
|
4
|
+
s.licenses = ['']
|
5
|
+
s.summary = "Stores logs in Elasticsearch"
|
6
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
|
+
s.authors = ["Elastic&Test"]
|
8
|
+
s.email = 'info@elastic.co'
|
9
|
+
s.homepage = "http://logstash.net/"
|
10
|
+
s.require_paths = ["lib"]
|
11
|
+
|
12
|
+
s.platform = RUBY_PLATFORM
|
13
|
+
|
14
|
+
# Files
|
15
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
16
|
+
|
17
|
+
# Tests
|
18
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
19
|
+
|
20
|
+
# Special flag to let us know this is actually a logstash plugin
|
21
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
22
|
+
|
23
|
+
s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
|
24
|
+
s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
|
25
|
+
s.add_runtime_dependency 'cabin', ['~> 0.6']
|
26
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
27
|
+
|
28
|
+
s.add_development_dependency 'logstash-codec-plain'
|
29
|
+
s.add_development_dependency 'logstash-devutils'
|
30
|
+
s.add_development_dependency 'flores'
|
31
|
+
# Still used in some specs, we should remove this ASAP
|
32
|
+
s.add_development_dependency 'elasticsearch'
|
33
|
+
end
|
@@ -0,0 +1,189 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require 'manticore'
|
3
|
+
require 'elasticsearch'
|
4
|
+
require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
|
5
|
+
require_relative "support/elasticsearch/api/actions/get_alias"
|
6
|
+
require_relative "support/elasticsearch/api/actions/put_alias"
|
7
|
+
require_relative "support/elasticsearch/api/actions/get_ilm_policy"
|
8
|
+
require_relative "support/elasticsearch/api/actions/put_ilm_policy"
|
9
|
+
|
10
|
+
require 'json'
|
11
|
+
|
12
|
+
unless defined?(LogStash::OSS)
|
13
|
+
LogStash::OSS = ENV['DISTRIBUTION'] != "default"
|
14
|
+
end
|
15
|
+
|
16
|
+
module ESHelper
|
17
|
+
def get_host_port
|
18
|
+
if ENV["INTEGRATION"] == "true"
|
19
|
+
"elasticsearch:9200"
|
20
|
+
else
|
21
|
+
"localhost:9200"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_client
|
26
|
+
Elasticsearch::Client.new(:hosts => [get_host_port])
|
27
|
+
end
|
28
|
+
|
29
|
+
def doc_type
|
30
|
+
if ESHelper.es_version_satisfies?(">=8")
|
31
|
+
nil
|
32
|
+
elsif ESHelper.es_version_satisfies?(">=7")
|
33
|
+
"_doc"
|
34
|
+
else
|
35
|
+
"doc"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.action_for_version(action)
|
40
|
+
action_params = action[1]
|
41
|
+
if ESHelper.es_version_satisfies?(">=8")
|
42
|
+
action_params.delete(:_type)
|
43
|
+
end
|
44
|
+
action[1] = action_params
|
45
|
+
action
|
46
|
+
end
|
47
|
+
|
48
|
+
def todays_date
|
49
|
+
Time.now.strftime("%Y.%m.%d")
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
def default_mapping_from_mappings(mappings)
|
54
|
+
if ESHelper.es_version_satisfies?(">=7")
|
55
|
+
mappings
|
56
|
+
else
|
57
|
+
mappings["_default_"]
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def field_properties_from_template(template_name, field)
|
62
|
+
mappings = @es.indices.get_template(name: template_name)[template_name]["mappings"]
|
63
|
+
mapping = default_mapping_from_mappings(mappings)
|
64
|
+
mapping["properties"][field]["properties"]
|
65
|
+
end
|
66
|
+
|
67
|
+
def routing_field_name
|
68
|
+
if ESHelper.es_version_satisfies?(">=6")
|
69
|
+
:routing
|
70
|
+
else
|
71
|
+
:_routing
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.es_version
|
76
|
+
RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
|
77
|
+
end
|
78
|
+
|
79
|
+
RSpec::Matchers.define :have_hits do |expected|
|
80
|
+
match do |actual|
|
81
|
+
if ESHelper.es_version_satisfies?(">=7")
|
82
|
+
expected == actual['hits']['total']['value']
|
83
|
+
else
|
84
|
+
expected == actual['hits']['total']
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
RSpec::Matchers.define :have_index_pattern do |expected|
|
90
|
+
match do |actual|
|
91
|
+
test_against = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
|
92
|
+
test_against.include?(expected)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.es_version_satisfies?(*requirement)
|
97
|
+
es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
|
98
|
+
if es_version.nil?
|
99
|
+
puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
|
100
|
+
return false
|
101
|
+
end
|
102
|
+
es_release_version = Gem::Version.new(es_version).release
|
103
|
+
Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
|
104
|
+
end
|
105
|
+
|
106
|
+
def clean(client)
|
107
|
+
client.indices.delete_template(:name => "*")
|
108
|
+
# This can fail if there are no indexes, ignore failure.
|
109
|
+
client.indices.delete(:index => "*") rescue nil
|
110
|
+
clean_ilm(client) if supports_ilm?(client)
|
111
|
+
end
|
112
|
+
|
113
|
+
def set_cluster_settings(client, cluster_settings)
|
114
|
+
client.cluster.put_settings(body: cluster_settings)
|
115
|
+
get_cluster_settings(client)
|
116
|
+
end
|
117
|
+
|
118
|
+
def get_cluster_settings(client)
|
119
|
+
client.cluster.get_settings
|
120
|
+
end
|
121
|
+
|
122
|
+
def get_policy(client, policy_name)
|
123
|
+
client.get_ilm_policy(name: policy_name)
|
124
|
+
end
|
125
|
+
|
126
|
+
def put_policy(client, policy_name, policy)
|
127
|
+
client.put_ilm_policy({:name => policy_name, :body=> policy})
|
128
|
+
end
|
129
|
+
|
130
|
+
def put_alias(client, the_alias, index)
|
131
|
+
body = {
|
132
|
+
"aliases" => {
|
133
|
+
index => {
|
134
|
+
"is_write_index"=> true
|
135
|
+
}
|
136
|
+
}
|
137
|
+
}
|
138
|
+
client.put_alias({name: the_alias, body: body})
|
139
|
+
end
|
140
|
+
|
141
|
+
def clean_ilm(client)
|
142
|
+
client.get_ilm_policy.each_key {|key| client.delete_ilm_policy(name: key)}
|
143
|
+
end
|
144
|
+
|
145
|
+
def supports_ilm?(client)
|
146
|
+
begin
|
147
|
+
client.get_ilm_policy
|
148
|
+
true
|
149
|
+
rescue
|
150
|
+
false
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def max_docs_policy(max_docs)
|
155
|
+
{
|
156
|
+
"policy" => {
|
157
|
+
"phases"=> {
|
158
|
+
"hot" => {
|
159
|
+
"actions" => {
|
160
|
+
"rollover" => {
|
161
|
+
"max_docs" => max_docs
|
162
|
+
}
|
163
|
+
}
|
164
|
+
}
|
165
|
+
}
|
166
|
+
}
|
167
|
+
}
|
168
|
+
end
|
169
|
+
|
170
|
+
def max_age_policy(max_age)
|
171
|
+
{
|
172
|
+
"policy" => {
|
173
|
+
"phases"=> {
|
174
|
+
"hot" => {
|
175
|
+
"actions" => {
|
176
|
+
"rollover" => {
|
177
|
+
"max_age" => max_age
|
178
|
+
}
|
179
|
+
}
|
180
|
+
}
|
181
|
+
}
|
182
|
+
}
|
183
|
+
}
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
RSpec.configure do |config|
|
188
|
+
config.include ESHelper
|
189
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
{
|
2
|
+
"cluster_name" : "dev",
|
3
|
+
"nodes" : {
|
4
|
+
"Ur_68iBvTlm7Xr1HgSsh6w" : {
|
5
|
+
"name" : "dev-es-master01",
|
6
|
+
"transport_address" : "http://localhost:9200",
|
7
|
+
"host" : "127.0.0.1",
|
8
|
+
"ip" : "127.0.0.1",
|
9
|
+
"version" : "2.4.6",
|
10
|
+
"build" : "5376dca"
|
11
|
+
},
|
12
|
+
"sari4do3RG-mgh2CIZeHwA" : {
|
13
|
+
"name" : "dev-es-data01",
|
14
|
+
"transport_address" : "http://localhost:9201",
|
15
|
+
"host" : "127.0.0.1",
|
16
|
+
"ip" : "127.0.0.1",
|
17
|
+
"version" : "2.4.6",
|
18
|
+
"build" : "5376dca",
|
19
|
+
"http_address" : "127.0.0.1:9201",
|
20
|
+
"http" : {
|
21
|
+
"bound_address" : [ "[::1]:9201", "127.0.0.1:9201" ],
|
22
|
+
"publish_address" : "127.0.0.1:9201",
|
23
|
+
"max_content_length_in_bytes" : 104857600
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
}
|
@@ -0,0 +1,81 @@
|
|
1
|
+
{
|
2
|
+
"_nodes" : {
|
3
|
+
"total" : 3,
|
4
|
+
"successful" : 3,
|
5
|
+
"failed" : 0
|
6
|
+
},
|
7
|
+
"cluster_name" : "dev",
|
8
|
+
"nodes" : {
|
9
|
+
"Ur_68iBvTlm7Xr1HgSsh6w" : {
|
10
|
+
"name" : "dev-es-master01",
|
11
|
+
"transport_address" : "http://localhost:9200",
|
12
|
+
"host" : "localhost",
|
13
|
+
"ip" : "127.0.0.1",
|
14
|
+
"version" : "5.5.1",
|
15
|
+
"build_hash" : "19c13d0",
|
16
|
+
"roles" : [
|
17
|
+
"master"
|
18
|
+
],
|
19
|
+
"http" : {
|
20
|
+
"bound_address" : [
|
21
|
+
"[::]:9200"
|
22
|
+
],
|
23
|
+
"publish_address" : "127.0.0.1:9200",
|
24
|
+
"max_content_length_in_bytes" : 104857600
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"sari4do3RG-mgh2CIZeHwA" : {
|
28
|
+
"name" : "dev-es-data01",
|
29
|
+
"transport_address" : "http://localhost:9201",
|
30
|
+
"host" : "localhost",
|
31
|
+
"ip" : "127.0.0.1",
|
32
|
+
"version" : "5.5.1",
|
33
|
+
"build_hash" : "19c13d0",
|
34
|
+
"roles" : [
|
35
|
+
"data"
|
36
|
+
],
|
37
|
+
"http" : {
|
38
|
+
"bound_address" : [
|
39
|
+
"[::]:9200"
|
40
|
+
],
|
41
|
+
"publish_address" : "127.0.0.1:9201",
|
42
|
+
"max_content_length_in_bytes" : 104857600
|
43
|
+
}
|
44
|
+
},
|
45
|
+
"Rjy1WL66RHm4fyzXA8PCGQ" : {
|
46
|
+
"name" : "dev-es-datamaster01",
|
47
|
+
"transport_address" : "http://localhost:9202",
|
48
|
+
"host" : "localhost",
|
49
|
+
"ip" : "127.0.0.1",
|
50
|
+
"version" : "5.5.1",
|
51
|
+
"build_hash" : "19c13d0",
|
52
|
+
"roles" : [
|
53
|
+
"data",
|
54
|
+
"master"
|
55
|
+
],
|
56
|
+
"http" : {
|
57
|
+
"bound_address" : [
|
58
|
+
"[::]:9200"
|
59
|
+
],
|
60
|
+
"publish_address" : "127.0.0.1:9202",
|
61
|
+
"max_content_length_in_bytes" : 104857600
|
62
|
+
}
|
63
|
+
},
|
64
|
+
"OguP_obcT_S9JYNB8SKKgQ" : {
|
65
|
+
"name" : "dev-es-coordinator01",
|
66
|
+
"transport_address" : "http://localhost:9203",
|
67
|
+
"host" : "localhost",
|
68
|
+
"ip" : "127.0.0.1",
|
69
|
+
"version" : "5.5.1",
|
70
|
+
"build_hash" : "19c13d0",
|
71
|
+
"roles" : [ ],
|
72
|
+
"http" : {
|
73
|
+
"bound_address" : [
|
74
|
+
"[::]:9200"
|
75
|
+
],
|
76
|
+
"publish_address" : "127.0.0.1:9203",
|
77
|
+
"max_content_length_in_bytes" : 104857600
|
78
|
+
}
|
79
|
+
}
|
80
|
+
}
|
81
|
+
}
|