logstash-output-elasticsearch-test 10.3.0-x86_64-linux
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +397 -0
- data/CONTRIBUTORS +33 -0
- data/Gemfile +15 -0
- data/LICENSE +13 -0
- data/NOTICE.TXT +5 -0
- data/README.md +106 -0
- data/docs/index.asciidoc +899 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +441 -0
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +167 -0
- data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json +95 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json +46 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json +45 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json +44 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json +44 -0
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +131 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +495 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +432 -0
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +159 -0
- data/lib/logstash/outputs/elasticsearch/ilm.rb +113 -0
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +61 -0
- data/lib/logstash/outputs/elasticsearch.rb +263 -0
- data/logstash-output-elasticsearch.gemspec +33 -0
- data/spec/es_spec_helper.rb +189 -0
- data/spec/fixtures/_nodes/2x_1x.json +27 -0
- data/spec/fixtures/_nodes/5x_6x.json +81 -0
- data/spec/fixtures/_nodes/7x.json +92 -0
- data/spec/fixtures/htpasswd +2 -0
- data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
- data/spec/fixtures/scripts/groovy/scripted_update.groovy +2 -0
- data/spec/fixtures/scripts/groovy/scripted_update_nested.groovy +2 -0
- data/spec/fixtures/scripts/groovy/scripted_upsert.groovy +2 -0
- data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
- data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
- data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
- data/spec/fixtures/template-with-policy-es6x.json +48 -0
- data/spec/fixtures/template-with-policy-es7x.json +45 -0
- data/spec/fixtures/test_certs/ca/ca.crt +32 -0
- data/spec/fixtures/test_certs/ca/ca.key +51 -0
- data/spec/fixtures/test_certs/test.crt +36 -0
- data/spec/fixtures/test_certs/test.key +51 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +69 -0
- data/spec/integration/outputs/create_spec.rb +67 -0
- data/spec/integration/outputs/delete_spec.rb +65 -0
- data/spec/integration/outputs/groovy_update_spec.rb +150 -0
- data/spec/integration/outputs/ilm_spec.rb +531 -0
- data/spec/integration/outputs/index_spec.rb +178 -0
- data/spec/integration/outputs/index_version_spec.rb +102 -0
- data/spec/integration/outputs/ingest_pipeline_spec.rb +74 -0
- data/spec/integration/outputs/metrics_spec.rb +70 -0
- data/spec/integration/outputs/no_es_on_startup_spec.rb +58 -0
- data/spec/integration/outputs/painless_update_spec.rb +189 -0
- data/spec/integration/outputs/parent_spec.rb +102 -0
- data/spec/integration/outputs/retry_spec.rb +169 -0
- data/spec/integration/outputs/routing_spec.rb +61 -0
- data/spec/integration/outputs/sniffer_spec.rb +133 -0
- data/spec/integration/outputs/templates_5x_spec.rb +98 -0
- data/spec/integration/outputs/templates_spec.rb +98 -0
- data/spec/integration/outputs/update_spec.rb +116 -0
- data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
- data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
- data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
- data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
- data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
- data/spec/unit/http_client_builder_spec.rb +185 -0
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +149 -0
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +274 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +250 -0
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +25 -0
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +72 -0
- data/spec/unit/outputs/elasticsearch_spec.rb +675 -0
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +82 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +54 -0
- metadata +300 -0
@@ -0,0 +1,61 @@
|
|
1
|
+
module LogStash; module Outputs; class ElasticSearch
|
2
|
+
class TemplateManager
|
3
|
+
# To be mixed into the elasticsearch plugin base
|
4
|
+
def self.install_template(plugin)
|
5
|
+
return unless plugin.manage_template
|
6
|
+
if plugin.template.nil?
|
7
|
+
plugin.logger.info("Using default mapping template")
|
8
|
+
else
|
9
|
+
plugin.logger.info("Using mapping template from", :path => plugin.template)
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
template = get_template(plugin.template, plugin.maximum_seen_major_version)
|
14
|
+
add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
|
15
|
+
plugin.logger.info("Attempting to install template", :manage_template => template)
|
16
|
+
install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
|
17
|
+
rescue => e
|
18
|
+
plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
def self.get_template(path, es_major_version)
|
23
|
+
template_path = path || default_template_path(es_major_version)
|
24
|
+
read_template_file(template_path)
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.install(client, template_name, template, template_overwrite)
|
28
|
+
client.template_install(template_name, template, template_overwrite)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.add_ilm_settings_to_template(plugin, template)
|
32
|
+
# Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern
|
33
|
+
# definition - remove any existing definition of 'template'
|
34
|
+
template.delete('template') if template.include?('template')
|
35
|
+
template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
|
36
|
+
if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
|
37
|
+
plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled.")
|
38
|
+
end
|
39
|
+
template['settings'].update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
|
40
|
+
end
|
41
|
+
|
42
|
+
# Template name - if template_name set, use it
|
43
|
+
# if not and ILM is enabled, use the rollover alias
|
44
|
+
# else use the default value of template_name
|
45
|
+
def self.template_name(plugin)
|
46
|
+
plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.default_template_path(es_major_version)
|
50
|
+
template_version = es_major_version == 1 ? 2 : es_major_version
|
51
|
+
default_template_name = "elasticsearch-template-es#{template_version}x.json"
|
52
|
+
::File.expand_path(default_template_name, ::File.dirname(__FILE__))
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.read_template_file(template_path)
|
56
|
+
raise ArgumentError, "Template file '#{template_path}' could not be found!" unless ::File.exists?(template_path)
|
57
|
+
template_data = ::IO.read(template_path)
|
58
|
+
LogStash::Json.load(template_data)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end end end
|
@@ -0,0 +1,263 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/namespace"
|
3
|
+
require "logstash/environment"
|
4
|
+
require "logstash/outputs/base"
|
5
|
+
require "logstash/json"
|
6
|
+
require "concurrent"
|
7
|
+
require "stud/buffer"
|
8
|
+
require "socket" # for Socket.gethostname
|
9
|
+
require "thread" # for safe queueing
|
10
|
+
require "uri" # for escaping user input
|
11
|
+
require "forwardable"
|
12
|
+
|
13
|
+
# .Compatibility Note
|
14
|
+
# [NOTE]
|
15
|
+
# ================================================================================
|
16
|
+
# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
|
17
|
+
# called `http.content_type.required`. If this option is set to `true`, and you
|
18
|
+
# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
|
19
|
+
# plugin to version 6.2.5 or higher.
|
20
|
+
#
|
21
|
+
# ================================================================================
|
22
|
+
#
|
23
|
+
# This plugin is the recommended method of storing logs in Elasticsearch.
|
24
|
+
# If you plan on using the Kibana web interface, you'll want to use this output.
|
25
|
+
#
|
26
|
+
# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
|
27
|
+
# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
|
28
|
+
# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
|
29
|
+
# to upgrade Logstash in lock-step.
|
30
|
+
#
|
31
|
+
# You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
|
32
|
+
#
|
33
|
+
# ==== Template management for Elasticsearch 5.x
|
34
|
+
# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
|
35
|
+
# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
|
36
|
+
# behavior.
|
37
|
+
#
|
38
|
+
# ** Users installing ES 5.x and LS 5.x **
|
39
|
+
# This change will not affect you and you will continue to use the ES defaults.
|
40
|
+
#
|
41
|
+
# ** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
|
42
|
+
# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
|
43
|
+
# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
|
44
|
+
# the new template is installed.
|
45
|
+
#
|
46
|
+
# ==== Retry Policy
|
47
|
+
#
|
48
|
+
# The retry policy has changed significantly in the 2.2.0 release.
|
49
|
+
# This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
|
50
|
+
# either partial or total failures.
|
51
|
+
#
|
52
|
+
# The following errors are retried infinitely:
|
53
|
+
#
|
54
|
+
# - Network errors (inability to connect)
|
55
|
+
# - 429 (Too many requests) and
|
56
|
+
# - 503 (Service unavailable) errors
|
57
|
+
#
|
58
|
+
# NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
|
59
|
+
# It is more performant for Elasticsearch to retry these exceptions than this plugin.
|
60
|
+
#
|
61
|
+
# ==== Batch Sizes ====
|
62
|
+
# This plugin attempts to send batches of events as a single request. However, if
|
63
|
+
# a request exceeds 20MB we will break it up until multiple batch requests. If a single document exceeds 20MB it will be sent as a single request.
|
64
|
+
#
|
65
|
+
# ==== DNS Caching
|
66
|
+
#
|
67
|
+
# This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
|
68
|
+
# a global setting for the JVM.
|
69
|
+
#
|
70
|
+
# As an example, to set your DNS TTL to 1 second you would set
|
71
|
+
# the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
|
72
|
+
#
|
73
|
+
# Keep in mind that a connection with keepalive enabled will
|
74
|
+
# not reevaluate its DNS value while the keepalive is in effect.
|
75
|
+
#
|
76
|
+
# ==== HTTP Compression
|
77
|
+
#
|
78
|
+
# This plugin supports request and response compression. Response compression is enabled by default and
|
79
|
+
# for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for
|
80
|
+
# it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` in
|
81
|
+
# Elasticsearch[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http] to take advantage of response compression when using this plugin
|
82
|
+
#
|
83
|
+
# For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression`
|
84
|
+
# setting in their Logstash config file.
|
85
|
+
#
|
86
|
+
class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
87
|
+
declare_threadsafe!
|
88
|
+
|
89
|
+
require "logstash/outputs/elasticsearch/http_client"
|
90
|
+
require "logstash/outputs/elasticsearch/http_client_builder"
|
91
|
+
require "logstash/outputs/elasticsearch/common_configs"
|
92
|
+
require "logstash/outputs/elasticsearch/common"
|
93
|
+
require "logstash/outputs/elasticsearch/ilm"
|
94
|
+
|
95
|
+
# Protocol agnostic (i.e. non-http, non-java specific) configs go here
|
96
|
+
include(LogStash::Outputs::ElasticSearch::CommonConfigs)
|
97
|
+
|
98
|
+
# Protocol agnostic methods
|
99
|
+
include(LogStash::Outputs::ElasticSearch::Common)
|
100
|
+
|
101
|
+
# Methods for ILM support
|
102
|
+
include(LogStash::Outputs::ElasticSearch::Ilm)
|
103
|
+
|
104
|
+
config_name "elasticsearch"
|
105
|
+
|
106
|
+
# The Elasticsearch action to perform. Valid actions are:
|
107
|
+
#
|
108
|
+
# - index: indexes a document (an event from Logstash).
|
109
|
+
# - delete: deletes a document by id (An id is required for this action)
|
110
|
+
# - create: indexes a document, fails if a document by that id already exists in the index.
|
111
|
+
# - update: updates a document by id. Update has a special case where you can upsert -- update a
|
112
|
+
# document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
|
113
|
+
# in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
|
114
|
+
# - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
|
115
|
+
# would use the foo field for the action
|
116
|
+
#
|
117
|
+
# For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
|
118
|
+
config :action, :validate => :string, :default => "index"
|
119
|
+
|
120
|
+
# Username to authenticate to a secure Elasticsearch cluster
|
121
|
+
config :user, :validate => :string
|
122
|
+
# Password to authenticate to a secure Elasticsearch cluster
|
123
|
+
config :password, :validate => :password
|
124
|
+
|
125
|
+
# Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
|
126
|
+
#
|
127
|
+
# For mode details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
|
128
|
+
config :cloud_auth, :validate => :password
|
129
|
+
|
130
|
+
# HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
|
131
|
+
# the root path for the Elasticsearch HTTP API lives.
|
132
|
+
# Note that if you use paths as components of URLs in the 'hosts' field you may
|
133
|
+
# not also set this field. That will raise an error at startup
|
134
|
+
config :path, :validate => :string
|
135
|
+
|
136
|
+
# HTTP Path to perform the _bulk requests to
|
137
|
+
# this defaults to a concatenation of the path parameter and "_bulk"
|
138
|
+
config :bulk_path, :validate => :string
|
139
|
+
|
140
|
+
# Pass a set of key value pairs as the URL query string. This query string is added
|
141
|
+
# to every host listed in the 'hosts' configuration. If the 'hosts' list contains
|
142
|
+
# urls that already have query strings, the one specified here will be appended.
|
143
|
+
config :parameters, :validate => :hash
|
144
|
+
|
145
|
+
# Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
|
146
|
+
# is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
|
147
|
+
# If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
|
148
|
+
config :ssl, :validate => :boolean
|
149
|
+
|
150
|
+
# Option to validate the server's certificate. Disabling this severely compromises security.
|
151
|
+
# For more information on disabling certificate verification please read
|
152
|
+
# https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
|
153
|
+
config :ssl_certificate_verification, :validate => :boolean, :default => true
|
154
|
+
|
155
|
+
# The .cer or .pem file to validate the server's certificate
|
156
|
+
config :cacert, :validate => :path
|
157
|
+
|
158
|
+
# The JKS truststore to validate the server's certificate.
|
159
|
+
# Use either `:truststore` or `:cacert`
|
160
|
+
config :truststore, :validate => :path
|
161
|
+
|
162
|
+
# Set the truststore password
|
163
|
+
config :truststore_password, :validate => :password
|
164
|
+
|
165
|
+
# The keystore used to present a certificate to the server.
|
166
|
+
# It can be either .jks or .p12
|
167
|
+
config :keystore, :validate => :path
|
168
|
+
|
169
|
+
# Set the keystore password
|
170
|
+
config :keystore_password, :validate => :password
|
171
|
+
|
172
|
+
# This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
|
173
|
+
# Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
|
174
|
+
# this with master nodes, you probably want to disable HTTP on them by setting
|
175
|
+
# `http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
|
176
|
+
# manually enter multiple Elasticsearch hosts using the `hosts` parameter.
|
177
|
+
config :sniffing, :validate => :boolean, :default => false
|
178
|
+
|
179
|
+
# How long to wait, in seconds, between sniffing attempts
|
180
|
+
config :sniffing_delay, :validate => :number, :default => 5
|
181
|
+
|
182
|
+
# HTTP Path to be used for the sniffing requests
|
183
|
+
# the default value is computed by concatenating the path value and "_nodes/http"
|
184
|
+
# if sniffing_path is set it will be used as an absolute path
|
185
|
+
# do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
|
186
|
+
config :sniffing_path, :validate => :string
|
187
|
+
|
188
|
+
# Set the address of a forward HTTP proxy.
|
189
|
+
# This used to accept hashes as arguments but now only accepts
|
190
|
+
# arguments of the URI type to prevent leaking credentials.
|
191
|
+
config :proxy, :validate => :uri
|
192
|
+
|
193
|
+
# Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
|
194
|
+
# a timeout occurs, the request will be retried.
|
195
|
+
config :timeout, :validate => :number, :default => 60
|
196
|
+
|
197
|
+
# Set the Elasticsearch errors in the whitelist that you don't want to log.
|
198
|
+
# A useful example is when you want to skip all 409 errors
|
199
|
+
# which are `document_already_exists_exception`.
|
200
|
+
config :failure_type_logging_whitelist, :validate => :array, :default => []
|
201
|
+
|
202
|
+
# While the output tries to reuse connections efficiently we have a maximum.
|
203
|
+
# This sets the maximum number of open connections the output will create.
|
204
|
+
# Setting this too low may mean frequently closing / opening connections
|
205
|
+
# which is bad.
|
206
|
+
config :pool_max, :validate => :number, :default => 1000
|
207
|
+
|
208
|
+
# While the output tries to reuse connections efficiently we have a maximum per endpoint.
|
209
|
+
# This sets the maximum number of open connections per endpoint the output will create.
|
210
|
+
# Setting this too low may mean frequently closing / opening connections
|
211
|
+
# which is bad.
|
212
|
+
config :pool_max_per_route, :validate => :number, :default => 100
|
213
|
+
|
214
|
+
# HTTP Path where a HEAD request is sent when a backend is marked down
|
215
|
+
# the request is sent in the background to see if it has come back again
|
216
|
+
# before it is once again eligible to service requests.
|
217
|
+
# If you have custom firewall rules you may need to change this
|
218
|
+
config :healthcheck_path, :validate => :string
|
219
|
+
|
220
|
+
# How frequently, in seconds, to wait between resurrection attempts.
|
221
|
+
# Resurrection is the process by which backend endpoints marked 'down' are checked
|
222
|
+
# to see if they have come back to life
|
223
|
+
config :resurrect_delay, :validate => :number, :default => 5
|
224
|
+
|
225
|
+
# How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
|
226
|
+
# You may want to set this lower, if you get connection errors regularly
|
227
|
+
# Quoting the Apache commons docs (this client is based Apache Commmons):
|
228
|
+
# 'Defines period of inactivity in milliseconds after which persistent connections must
|
229
|
+
# be re-validated prior to being leased to the consumer. Non-positive value passed to
|
230
|
+
# this method disables connection validation. This check helps detect connections that
|
231
|
+
# have become stale (half-closed) while kept inactive in the pool.'
|
232
|
+
# See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
|
233
|
+
config :validate_after_inactivity, :validate => :number, :default => 10000
|
234
|
+
|
235
|
+
# Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
|
236
|
+
config :http_compression, :validate => :boolean, :default => false
|
237
|
+
|
238
|
+
# Custom Headers to send on each request to elasticsearch nodes
|
239
|
+
config :custom_headers, :validate => :hash, :default => {}
|
240
|
+
|
241
|
+
def build_client
|
242
|
+
params["metric"] = metric
|
243
|
+
@client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
|
244
|
+
end
|
245
|
+
|
246
|
+
def close
|
247
|
+
@stopping.make_true
|
248
|
+
stop_template_installer
|
249
|
+
@client.close if @client
|
250
|
+
end
|
251
|
+
|
252
|
+
def self.oss?
|
253
|
+
LogStash::OSS
|
254
|
+
end
|
255
|
+
|
256
|
+
@@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
|
257
|
+
|
258
|
+
@@plugins.each do |plugin|
|
259
|
+
name = plugin.name.split('-')[-1]
|
260
|
+
require "logstash/outputs/elasticsearch/#{name}"
|
261
|
+
end
|
262
|
+
|
263
|
+
end # class LogStash::Outputs::Elasticsearch
|
@@ -0,0 +1,33 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-elasticsearch-test'
|
3
|
+
s.version = '10.3.0'
|
4
|
+
s.licenses = ['']
|
5
|
+
s.summary = "Stores logs in Elasticsearch"
|
6
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
|
+
s.authors = ["Elastic&Test"]
|
8
|
+
s.email = 'info@elastic.co'
|
9
|
+
s.homepage = "http://logstash.net/"
|
10
|
+
s.require_paths = ["lib"]
|
11
|
+
|
12
|
+
s.platform = RUBY_PLATFORM
|
13
|
+
|
14
|
+
# Files
|
15
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
16
|
+
|
17
|
+
# Tests
|
18
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
19
|
+
|
20
|
+
# Special flag to let us know this is actually a logstash plugin
|
21
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
22
|
+
|
23
|
+
s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
|
24
|
+
s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
|
25
|
+
s.add_runtime_dependency 'cabin', ['~> 0.6']
|
26
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
27
|
+
|
28
|
+
s.add_development_dependency 'logstash-codec-plain'
|
29
|
+
s.add_development_dependency 'logstash-devutils'
|
30
|
+
s.add_development_dependency 'flores'
|
31
|
+
# Still used in some specs, we should remove this ASAP
|
32
|
+
s.add_development_dependency 'elasticsearch'
|
33
|
+
end
|
@@ -0,0 +1,189 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require 'manticore'
|
3
|
+
require 'elasticsearch'
|
4
|
+
require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
|
5
|
+
require_relative "support/elasticsearch/api/actions/get_alias"
|
6
|
+
require_relative "support/elasticsearch/api/actions/put_alias"
|
7
|
+
require_relative "support/elasticsearch/api/actions/get_ilm_policy"
|
8
|
+
require_relative "support/elasticsearch/api/actions/put_ilm_policy"
|
9
|
+
|
10
|
+
require 'json'
|
11
|
+
|
12
|
+
unless defined?(LogStash::OSS)
|
13
|
+
LogStash::OSS = ENV['DISTRIBUTION'] != "default"
|
14
|
+
end
|
15
|
+
|
16
|
+
module ESHelper
|
17
|
+
def get_host_port
|
18
|
+
if ENV["INTEGRATION"] == "true"
|
19
|
+
"elasticsearch:9200"
|
20
|
+
else
|
21
|
+
"localhost:9200"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_client
|
26
|
+
Elasticsearch::Client.new(:hosts => [get_host_port])
|
27
|
+
end
|
28
|
+
|
29
|
+
def doc_type
|
30
|
+
if ESHelper.es_version_satisfies?(">=8")
|
31
|
+
nil
|
32
|
+
elsif ESHelper.es_version_satisfies?(">=7")
|
33
|
+
"_doc"
|
34
|
+
else
|
35
|
+
"doc"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.action_for_version(action)
|
40
|
+
action_params = action[1]
|
41
|
+
if ESHelper.es_version_satisfies?(">=8")
|
42
|
+
action_params.delete(:_type)
|
43
|
+
end
|
44
|
+
action[1] = action_params
|
45
|
+
action
|
46
|
+
end
|
47
|
+
|
48
|
+
def todays_date
|
49
|
+
Time.now.strftime("%Y.%m.%d")
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
def default_mapping_from_mappings(mappings)
|
54
|
+
if ESHelper.es_version_satisfies?(">=7")
|
55
|
+
mappings
|
56
|
+
else
|
57
|
+
mappings["_default_"]
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def field_properties_from_template(template_name, field)
|
62
|
+
mappings = @es.indices.get_template(name: template_name)[template_name]["mappings"]
|
63
|
+
mapping = default_mapping_from_mappings(mappings)
|
64
|
+
mapping["properties"][field]["properties"]
|
65
|
+
end
|
66
|
+
|
67
|
+
def routing_field_name
|
68
|
+
if ESHelper.es_version_satisfies?(">=6")
|
69
|
+
:routing
|
70
|
+
else
|
71
|
+
:_routing
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.es_version
|
76
|
+
RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
|
77
|
+
end
|
78
|
+
|
79
|
+
RSpec::Matchers.define :have_hits do |expected|
|
80
|
+
match do |actual|
|
81
|
+
if ESHelper.es_version_satisfies?(">=7")
|
82
|
+
expected == actual['hits']['total']['value']
|
83
|
+
else
|
84
|
+
expected == actual['hits']['total']
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
RSpec::Matchers.define :have_index_pattern do |expected|
|
90
|
+
match do |actual|
|
91
|
+
test_against = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
|
92
|
+
test_against.include?(expected)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def self.es_version_satisfies?(*requirement)
|
97
|
+
es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
|
98
|
+
if es_version.nil?
|
99
|
+
puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
|
100
|
+
return false
|
101
|
+
end
|
102
|
+
es_release_version = Gem::Version.new(es_version).release
|
103
|
+
Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
|
104
|
+
end
|
105
|
+
|
106
|
+
def clean(client)
|
107
|
+
client.indices.delete_template(:name => "*")
|
108
|
+
# This can fail if there are no indexes, ignore failure.
|
109
|
+
client.indices.delete(:index => "*") rescue nil
|
110
|
+
clean_ilm(client) if supports_ilm?(client)
|
111
|
+
end
|
112
|
+
|
113
|
+
def set_cluster_settings(client, cluster_settings)
|
114
|
+
client.cluster.put_settings(body: cluster_settings)
|
115
|
+
get_cluster_settings(client)
|
116
|
+
end
|
117
|
+
|
118
|
+
def get_cluster_settings(client)
|
119
|
+
client.cluster.get_settings
|
120
|
+
end
|
121
|
+
|
122
|
+
def get_policy(client, policy_name)
|
123
|
+
client.get_ilm_policy(name: policy_name)
|
124
|
+
end
|
125
|
+
|
126
|
+
def put_policy(client, policy_name, policy)
|
127
|
+
client.put_ilm_policy({:name => policy_name, :body=> policy})
|
128
|
+
end
|
129
|
+
|
130
|
+
def put_alias(client, the_alias, index)
|
131
|
+
body = {
|
132
|
+
"aliases" => {
|
133
|
+
index => {
|
134
|
+
"is_write_index"=> true
|
135
|
+
}
|
136
|
+
}
|
137
|
+
}
|
138
|
+
client.put_alias({name: the_alias, body: body})
|
139
|
+
end
|
140
|
+
|
141
|
+
def clean_ilm(client)
|
142
|
+
client.get_ilm_policy.each_key {|key| client.delete_ilm_policy(name: key)}
|
143
|
+
end
|
144
|
+
|
145
|
+
def supports_ilm?(client)
|
146
|
+
begin
|
147
|
+
client.get_ilm_policy
|
148
|
+
true
|
149
|
+
rescue
|
150
|
+
false
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def max_docs_policy(max_docs)
|
155
|
+
{
|
156
|
+
"policy" => {
|
157
|
+
"phases"=> {
|
158
|
+
"hot" => {
|
159
|
+
"actions" => {
|
160
|
+
"rollover" => {
|
161
|
+
"max_docs" => max_docs
|
162
|
+
}
|
163
|
+
}
|
164
|
+
}
|
165
|
+
}
|
166
|
+
}
|
167
|
+
}
|
168
|
+
end
|
169
|
+
|
170
|
+
def max_age_policy(max_age)
|
171
|
+
{
|
172
|
+
"policy" => {
|
173
|
+
"phases"=> {
|
174
|
+
"hot" => {
|
175
|
+
"actions" => {
|
176
|
+
"rollover" => {
|
177
|
+
"max_age" => max_age
|
178
|
+
}
|
179
|
+
}
|
180
|
+
}
|
181
|
+
}
|
182
|
+
}
|
183
|
+
}
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
RSpec.configure do |config|
|
188
|
+
config.include ESHelper
|
189
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
{
|
2
|
+
"cluster_name" : "dev",
|
3
|
+
"nodes" : {
|
4
|
+
"Ur_68iBvTlm7Xr1HgSsh6w" : {
|
5
|
+
"name" : "dev-es-master01",
|
6
|
+
"transport_address" : "http://localhost:9200",
|
7
|
+
"host" : "127.0.0.1",
|
8
|
+
"ip" : "127.0.0.1",
|
9
|
+
"version" : "2.4.6",
|
10
|
+
"build" : "5376dca"
|
11
|
+
},
|
12
|
+
"sari4do3RG-mgh2CIZeHwA" : {
|
13
|
+
"name" : "dev-es-data01",
|
14
|
+
"transport_address" : "http://localhost:9201",
|
15
|
+
"host" : "127.0.0.1",
|
16
|
+
"ip" : "127.0.0.1",
|
17
|
+
"version" : "2.4.6",
|
18
|
+
"build" : "5376dca",
|
19
|
+
"http_address" : "127.0.0.1:9201",
|
20
|
+
"http" : {
|
21
|
+
"bound_address" : [ "[::1]:9201", "127.0.0.1:9201" ],
|
22
|
+
"publish_address" : "127.0.0.1:9201",
|
23
|
+
"max_content_length_in_bytes" : 104857600
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
}
|
@@ -0,0 +1,81 @@
|
|
1
|
+
{
|
2
|
+
"_nodes" : {
|
3
|
+
"total" : 3,
|
4
|
+
"successful" : 3,
|
5
|
+
"failed" : 0
|
6
|
+
},
|
7
|
+
"cluster_name" : "dev",
|
8
|
+
"nodes" : {
|
9
|
+
"Ur_68iBvTlm7Xr1HgSsh6w" : {
|
10
|
+
"name" : "dev-es-master01",
|
11
|
+
"transport_address" : "http://localhost:9200",
|
12
|
+
"host" : "localhost",
|
13
|
+
"ip" : "127.0.0.1",
|
14
|
+
"version" : "5.5.1",
|
15
|
+
"build_hash" : "19c13d0",
|
16
|
+
"roles" : [
|
17
|
+
"master"
|
18
|
+
],
|
19
|
+
"http" : {
|
20
|
+
"bound_address" : [
|
21
|
+
"[::]:9200"
|
22
|
+
],
|
23
|
+
"publish_address" : "127.0.0.1:9200",
|
24
|
+
"max_content_length_in_bytes" : 104857600
|
25
|
+
}
|
26
|
+
},
|
27
|
+
"sari4do3RG-mgh2CIZeHwA" : {
|
28
|
+
"name" : "dev-es-data01",
|
29
|
+
"transport_address" : "http://localhost:9201",
|
30
|
+
"host" : "localhost",
|
31
|
+
"ip" : "127.0.0.1",
|
32
|
+
"version" : "5.5.1",
|
33
|
+
"build_hash" : "19c13d0",
|
34
|
+
"roles" : [
|
35
|
+
"data"
|
36
|
+
],
|
37
|
+
"http" : {
|
38
|
+
"bound_address" : [
|
39
|
+
"[::]:9200"
|
40
|
+
],
|
41
|
+
"publish_address" : "127.0.0.1:9201",
|
42
|
+
"max_content_length_in_bytes" : 104857600
|
43
|
+
}
|
44
|
+
},
|
45
|
+
"Rjy1WL66RHm4fyzXA8PCGQ" : {
|
46
|
+
"name" : "dev-es-datamaster01",
|
47
|
+
"transport_address" : "http://localhost:9202",
|
48
|
+
"host" : "localhost",
|
49
|
+
"ip" : "127.0.0.1",
|
50
|
+
"version" : "5.5.1",
|
51
|
+
"build_hash" : "19c13d0",
|
52
|
+
"roles" : [
|
53
|
+
"data",
|
54
|
+
"master"
|
55
|
+
],
|
56
|
+
"http" : {
|
57
|
+
"bound_address" : [
|
58
|
+
"[::]:9200"
|
59
|
+
],
|
60
|
+
"publish_address" : "127.0.0.1:9202",
|
61
|
+
"max_content_length_in_bytes" : 104857600
|
62
|
+
}
|
63
|
+
},
|
64
|
+
"OguP_obcT_S9JYNB8SKKgQ" : {
|
65
|
+
"name" : "dev-es-coordinator01",
|
66
|
+
"transport_address" : "http://localhost:9203",
|
67
|
+
"host" : "localhost",
|
68
|
+
"ip" : "127.0.0.1",
|
69
|
+
"version" : "5.5.1",
|
70
|
+
"build_hash" : "19c13d0",
|
71
|
+
"roles" : [ ],
|
72
|
+
"http" : {
|
73
|
+
"bound_address" : [
|
74
|
+
"[::]:9200"
|
75
|
+
],
|
76
|
+
"publish_address" : "127.0.0.1:9203",
|
77
|
+
"max_content_length_in_bytes" : 104857600
|
78
|
+
}
|
79
|
+
}
|
80
|
+
}
|
81
|
+
}
|