logstash-output-elasticsearch 10.7.3-java → 10.8.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/outputs/elasticsearch.rb +300 -165
- data/lib/logstash/outputs/elasticsearch/http_client.rb +1 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +13 -28
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +1 -0
- data/lib/logstash/outputs/elasticsearch/ilm.rb +9 -5
- data/lib/logstash/outputs/elasticsearch/license_checker.rb +47 -0
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +163 -0
- data/lib/logstash/{outputs → plugin_mixins}/elasticsearch/common.rb +40 -167
- data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +45 -5
- data/spec/unit/outputs/license_check_spec.rb +41 -0
- metadata +8 -4
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +0 -167
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 29d2b5e316c2ba3dd1d67af673733bd90dd0b5b82c436b1f107cc11d681bd8b5
|
4
|
+
data.tar.gz: 102c70fde50016c59245287f09eeed0d51d4fb411a617b7d397ab306688f022d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6bd3b56c08debf7408ac1cd31980235d3224b5705a42ecc78d28e8da94825cd310ee1b8d0f47b16400327bcc6b0c1e777e2a937a09b1a5a483584d6ca523317d
|
7
|
+
data.tar.gz: c9ba096fb1a39064616ef10be681d1d3b64ab344a7b3b51560b68391c525cf61c6c98a289e068958e43806e7042d9a7ced140ec29407f80680e0323ba44ce165
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 10.8.0
|
2
|
+
- Refactored configuration options into specific and shared in PluginMixins namespace [#973](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/973)
|
3
|
+
- Refactored common methods into specific and shared in PluginMixins namespace [#976](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/976)
|
4
|
+
|
1
5
|
## 10.7.3
|
2
6
|
- Added composable index template support for elasticsearch version 8 [#980](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/980)
|
3
7
|
|
@@ -86,19 +86,16 @@ require "forwardable"
|
|
86
86
|
class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
87
87
|
declare_threadsafe!
|
88
88
|
|
89
|
+
require "logstash/outputs/elasticsearch/license_checker"
|
89
90
|
require "logstash/outputs/elasticsearch/http_client"
|
90
91
|
require "logstash/outputs/elasticsearch/http_client_builder"
|
91
|
-
require "logstash/
|
92
|
-
require "logstash/
|
92
|
+
require "logstash/plugin_mixins/elasticsearch/api_configs"
|
93
|
+
require "logstash/plugin_mixins/elasticsearch/common"
|
93
94
|
require "logstash/outputs/elasticsearch/ilm"
|
94
|
-
|
95
95
|
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
96
96
|
|
97
|
-
# Protocol agnostic (i.e. non-http, non-java specific) configs go here
|
98
|
-
include(LogStash::Outputs::ElasticSearch::CommonConfigs)
|
99
|
-
|
100
97
|
# Protocol agnostic methods
|
101
|
-
include(LogStash::
|
98
|
+
include(LogStash::PluginMixins::ElasticSearch::Common)
|
102
99
|
|
103
100
|
# Methods for ILM support
|
104
101
|
include(LogStash::Outputs::ElasticSearch::Ilm)
|
@@ -106,6 +103,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
106
103
|
# ecs_compatibility option, provided by Logstash core or the support adapter.
|
107
104
|
include(LogStash::PluginMixins::ECSCompatibilitySupport)
|
108
105
|
|
106
|
+
# Generic/API config options that any document indexer output needs
|
107
|
+
include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
|
108
|
+
|
109
|
+
DEFAULT_POLICY = "logstash-policy"
|
110
|
+
|
109
111
|
config_name "elasticsearch"
|
110
112
|
|
111
113
|
# The Elasticsearch action to perform. Valid actions are:
|
@@ -122,158 +124,160 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
122
124
|
# For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
|
123
125
|
config :action, :validate => :string, :default => "index"
|
124
126
|
|
125
|
-
#
|
126
|
-
|
127
|
-
#
|
128
|
-
|
127
|
+
# The index to write events to. This can be dynamic using the `%{foo}` syntax.
|
128
|
+
# The default value will partition your indices by day so you can more easily
|
129
|
+
# delete old data or only search specific date ranges.
|
130
|
+
# Indexes may not contain uppercase characters.
|
131
|
+
# For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
|
132
|
+
# LS uses Joda to format the index pattern from event timestamp.
|
133
|
+
# Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
|
134
|
+
config :index, :validate => :string
|
135
|
+
|
136
|
+
config :document_type,
|
137
|
+
:validate => :string,
|
138
|
+
:deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
|
139
|
+
|
140
|
+
# From Logstash 1.3 onwards, a template is applied to Elasticsearch during
|
141
|
+
# Logstash's startup if one with the name `template_name` does not already exist.
|
142
|
+
# By default, the contents of this template is the default template for
|
143
|
+
# `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
|
144
|
+
# `logstash-*`. Should you require support for other index names, or would like
|
145
|
+
# to change the mappings in the template in general, a custom template can be
|
146
|
+
# specified by setting `template` to the path of a template file.
|
147
|
+
#
|
148
|
+
# Setting `manage_template` to false disables this feature. If you require more
|
149
|
+
# control over template creation, (e.g. creating indices dynamically based on
|
150
|
+
# field names) you should set `manage_template` to false and use the REST
|
151
|
+
# API to apply your templates manually.
|
152
|
+
config :manage_template, :validate => :boolean, :default => true
|
153
|
+
|
154
|
+
# This configuration option defines how the template is named inside Elasticsearch.
|
155
|
+
# Note that if you have used the template management features and subsequently
|
156
|
+
# change this, you will need to prune the old template manually, e.g.
|
157
|
+
#
|
158
|
+
# `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
|
159
|
+
#
|
160
|
+
# where `OldTemplateName` is whatever the former setting was.
|
161
|
+
config :template_name, :validate => :string
|
162
|
+
|
163
|
+
# You can set the path to your own template here, if you so desire.
|
164
|
+
# If not set, the included template will be used.
|
165
|
+
config :template, :validate => :path
|
166
|
+
|
167
|
+
# The template_overwrite option will always overwrite the indicated template
|
168
|
+
# in Elasticsearch with either the one indicated by template or the included one.
|
169
|
+
# This option is set to false by default. If you always want to stay up to date
|
170
|
+
# with the template provided by Logstash, this option could be very useful to you.
|
171
|
+
# Likewise, if you have your own template file managed by puppet, for example, and
|
172
|
+
# you wanted to be able to update it regularly, this option could help there as well.
|
173
|
+
#
|
174
|
+
# Please note that if you are using your own customized version of the Logstash
|
175
|
+
# template (logstash), setting this to true will make Logstash to overwrite
|
176
|
+
# the "logstash" template (i.e. removing all customized settings)
|
177
|
+
config :template_overwrite, :validate => :boolean, :default => false
|
129
178
|
|
130
|
-
#
|
131
|
-
#
|
132
|
-
config :
|
179
|
+
# The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
|
180
|
+
# See https://www.elastic.co/blog/elasticsearch-versioning-support.
|
181
|
+
config :version, :validate => :string
|
133
182
|
|
134
|
-
#
|
135
|
-
#
|
136
|
-
#
|
137
|
-
config :
|
138
|
-
|
139
|
-
#
|
140
|
-
#
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
#
|
151
|
-
#
|
152
|
-
config :
|
153
|
-
|
154
|
-
# Enable
|
155
|
-
#
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
#
|
162
|
-
|
163
|
-
|
164
|
-
#
|
165
|
-
config :
|
166
|
-
|
167
|
-
#
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
#
|
179
|
-
config :
|
180
|
-
|
181
|
-
#
|
182
|
-
#
|
183
|
-
|
184
|
-
|
185
|
-
#
|
186
|
-
|
187
|
-
|
188
|
-
#
|
189
|
-
config :
|
190
|
-
|
191
|
-
#
|
192
|
-
|
193
|
-
|
194
|
-
#
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
#
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
config :timeout, :validate => :number, :default => 60
|
205
|
-
|
206
|
-
# Set the Elasticsearch errors in the whitelist that you don't want to log.
|
207
|
-
# A useful example is when you want to skip all 409 errors
|
208
|
-
# which are `document_already_exists_exception`.
|
209
|
-
config :failure_type_logging_whitelist, :validate => :array, :default => []
|
210
|
-
|
211
|
-
# While the output tries to reuse connections efficiently we have a maximum.
|
212
|
-
# This sets the maximum number of open connections the output will create.
|
213
|
-
# Setting this too low may mean frequently closing / opening connections
|
214
|
-
# which is bad.
|
215
|
-
config :pool_max, :validate => :number, :default => 1000
|
216
|
-
|
217
|
-
# While the output tries to reuse connections efficiently we have a maximum per endpoint.
|
218
|
-
# This sets the maximum number of open connections per endpoint the output will create.
|
219
|
-
# Setting this too low may mean frequently closing / opening connections
|
220
|
-
# which is bad.
|
221
|
-
config :pool_max_per_route, :validate => :number, :default => 100
|
222
|
-
|
223
|
-
# HTTP Path where a HEAD request is sent when a backend is marked down
|
224
|
-
# the request is sent in the background to see if it has come back again
|
225
|
-
# before it is once again eligible to service requests.
|
226
|
-
# If you have custom firewall rules you may need to change this
|
227
|
-
config :healthcheck_path, :validate => :string
|
228
|
-
|
229
|
-
# How frequently, in seconds, to wait between resurrection attempts.
|
230
|
-
# Resurrection is the process by which backend endpoints marked 'down' are checked
|
231
|
-
# to see if they have come back to life
|
232
|
-
config :resurrect_delay, :validate => :number, :default => 5
|
233
|
-
|
234
|
-
# How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
|
235
|
-
# You may want to set this lower, if you get connection errors regularly
|
236
|
-
# Quoting the Apache commons docs (this client is based Apache Commmons):
|
237
|
-
# 'Defines period of inactivity in milliseconds after which persistent connections must
|
238
|
-
# be re-validated prior to being leased to the consumer. Non-positive value passed to
|
239
|
-
# this method disables connection validation. This check helps detect connections that
|
240
|
-
# have become stale (half-closed) while kept inactive in the pool.'
|
241
|
-
# See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
|
242
|
-
config :validate_after_inactivity, :validate => :number, :default => 10000
|
243
|
-
|
244
|
-
# Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
|
245
|
-
config :http_compression, :validate => :boolean, :default => false
|
246
|
-
|
247
|
-
# Custom Headers to send on each request to elasticsearch nodes
|
248
|
-
config :custom_headers, :validate => :hash, :default => {}
|
183
|
+
# The version_type to use for indexing.
|
184
|
+
# See https://www.elastic.co/blog/elasticsearch-versioning-support.
|
185
|
+
# See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#_version_types
|
186
|
+
config :version_type, :validate => ["internal", 'external', "external_gt", "external_gte", "force"]
|
187
|
+
|
188
|
+
# A routing override to be applied to all processed events.
|
189
|
+
# This can be dynamic using the `%{foo}` syntax.
|
190
|
+
config :routing, :validate => :string
|
191
|
+
|
192
|
+
# For child documents, ID of the associated parent.
|
193
|
+
# This can be dynamic using the `%{foo}` syntax.
|
194
|
+
config :parent, :validate => :string, :default => nil
|
195
|
+
|
196
|
+
# For child documents, name of the join field
|
197
|
+
config :join_field, :validate => :string, :default => nil
|
198
|
+
|
199
|
+
# Set upsert content for update mode.s
|
200
|
+
# Create a new document with this parameter as json string if `document_id` doesn't exists
|
201
|
+
config :upsert, :validate => :string, :default => ""
|
202
|
+
|
203
|
+
# Enable `doc_as_upsert` for update mode.
|
204
|
+
# Create a new document with source if `document_id` doesn't exist in Elasticsearch
|
205
|
+
config :doc_as_upsert, :validate => :boolean, :default => false
|
206
|
+
|
207
|
+
# Set script name for scripted update mode
|
208
|
+
config :script, :validate => :string, :default => ""
|
209
|
+
|
210
|
+
# Define the type of script referenced by "script" variable
|
211
|
+
# inline : "script" contains inline script
|
212
|
+
# indexed : "script" contains the name of script directly indexed in elasticsearch
|
213
|
+
# file : "script" contains the name of script stored in elasticseach's config directory
|
214
|
+
config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
|
215
|
+
|
216
|
+
# Set the language of the used script. If not set, this defaults to painless in ES 5.0
|
217
|
+
config :script_lang, :validate => :string, :default => "painless"
|
218
|
+
|
219
|
+
# Set variable name passed to script (scripted update)
|
220
|
+
config :script_var_name, :validate => :string, :default => "event"
|
221
|
+
|
222
|
+
# if enabled, script is in charge of creating non-existent document (scripted update)
|
223
|
+
config :scripted_upsert, :validate => :boolean, :default => false
|
224
|
+
|
225
|
+
# The number of times Elasticsearch should internally retry an update/upserted document
|
226
|
+
# See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
|
227
|
+
# for more info
|
228
|
+
config :retry_on_conflict, :validate => :number, :default => 1
|
229
|
+
|
230
|
+
# Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
|
231
|
+
# here like `pipeline => "%{INGEST_PIPELINE}"`
|
232
|
+
config :pipeline, :validate => :string, :default => nil
|
233
|
+
|
234
|
+
# -----
|
235
|
+
# ILM configurations (beta)
|
236
|
+
# -----
|
237
|
+
# Flag for enabling Index Lifecycle Management integration.
|
238
|
+
config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto'
|
239
|
+
|
240
|
+
# Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
|
241
|
+
config :ilm_rollover_alias, :validate => :string
|
242
|
+
|
243
|
+
# appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
|
244
|
+
# {now/d} is date math, and will insert the appropriate value automatically.
|
245
|
+
config :ilm_pattern, :validate => :string, :default => '{now/d}-000001'
|
246
|
+
|
247
|
+
# ILM policy to use, if undefined the default policy will be used.
|
248
|
+
config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
|
249
|
+
|
250
|
+
attr_reader :default_index
|
251
|
+
attr_reader :default_ilm_rollover_alias
|
252
|
+
attr_reader :default_template_name
|
249
253
|
|
250
254
|
def initialize(*params)
|
251
255
|
super
|
252
256
|
setup_ecs_compatibility_related_defaults
|
253
257
|
end
|
254
258
|
|
255
|
-
def
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
@default_template_name = 'logstash'
|
261
|
-
when :v1
|
262
|
-
@default_index = "ecs-logstash-%{+yyyy.MM.dd}"
|
263
|
-
@default_ilm_rollover_alias = "ecs-logstash"
|
264
|
-
@default_template_name = 'ecs-logstash'
|
265
|
-
else
|
266
|
-
fail("unsupported ECS Compatibility `#{ecs_compatibility}`")
|
267
|
-
end
|
259
|
+
def register
|
260
|
+
@template_installed = Concurrent::AtomicBoolean.new(false)
|
261
|
+
@stopping = Concurrent::AtomicBoolean.new(false)
|
262
|
+
# To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
|
263
|
+
@dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
|
268
264
|
|
269
|
-
|
270
|
-
@ilm_rollover_alias ||= default_ilm_rollover_alias
|
271
|
-
@template_name ||= default_template_name
|
272
|
-
end
|
265
|
+
check_action_validity
|
273
266
|
|
274
|
-
|
275
|
-
|
276
|
-
|
267
|
+
# the license_checking behaviour in the Pool class is externalized in the LogStash::ElasticSearchOutputLicenseChecker
|
268
|
+
# class defined in license_check.rb. This license checking is specific to the elasticsearch output here and passed
|
269
|
+
# to build_client down to the Pool class.
|
270
|
+
build_client(LicenseChecker.new(@logger))
|
271
|
+
|
272
|
+
@template_installer = setup_after_successful_connection do
|
273
|
+
discover_cluster_uuid
|
274
|
+
install_template
|
275
|
+
setup_ilm if ilm_in_use?
|
276
|
+
end
|
277
|
+
@bulk_request_metrics = metric.namespace(:bulk_requests)
|
278
|
+
@document_level_metrics = metric.namespace(:documents)
|
279
|
+
@logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
|
280
|
+
end
|
277
281
|
|
278
282
|
# @override to handle proxy => '' as if none was set
|
279
283
|
def config_init(params)
|
@@ -291,20 +295,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
291
295
|
super(params)
|
292
296
|
end
|
293
297
|
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
# see https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/934#pullrequestreview-396203307
|
299
|
-
validate_authentication
|
300
|
-
fill_hosts_from_cloud_id
|
301
|
-
setup_hosts
|
302
|
-
|
303
|
-
params["metric"] = metric
|
304
|
-
if @proxy.eql?('')
|
305
|
-
@logger.warn "Supplied proxy setting (proxy => '') has no effect"
|
298
|
+
# Receive an array of events and immediately attempt to index them (no buffering)
|
299
|
+
def multi_receive(events)
|
300
|
+
until @template_installed.true?
|
301
|
+
sleep 1
|
306
302
|
end
|
307
|
-
|
303
|
+
retrying_submit(events.map {|e| event_action_tuple(e)})
|
308
304
|
end
|
309
305
|
|
310
306
|
def close
|
@@ -313,8 +309,65 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
313
309
|
@client.close if @client
|
314
310
|
end
|
315
311
|
|
316
|
-
|
317
|
-
|
312
|
+
# not private because used by ILM specs
|
313
|
+
def stop_template_installer
|
314
|
+
@template_installer.join unless @template_installer.nil?
|
315
|
+
end
|
316
|
+
|
317
|
+
# not private for elasticsearch_spec.rb
|
318
|
+
# Convert the event into a 3-tuple of action, params, and event
|
319
|
+
def event_action_tuple(event)
|
320
|
+
action = event.sprintf(@action)
|
321
|
+
|
322
|
+
params = {
|
323
|
+
:_id => @document_id ? event.sprintf(@document_id) : nil,
|
324
|
+
:_index => event.sprintf(@index),
|
325
|
+
routing_field_name => @routing ? event.sprintf(@routing) : nil
|
326
|
+
}
|
327
|
+
|
328
|
+
params[:_type] = get_event_type(event) if use_event_type?(nil)
|
329
|
+
|
330
|
+
if @pipeline
|
331
|
+
value = event.sprintf(@pipeline)
|
332
|
+
# convention: empty string equates to not using a pipeline
|
333
|
+
# this is useful when using a field reference in the pipeline setting, e.g.
|
334
|
+
# elasticsearch {
|
335
|
+
# pipeline => "%{[@metadata][pipeline]}"
|
336
|
+
# }
|
337
|
+
params[:pipeline] = value unless value.empty?
|
338
|
+
end
|
339
|
+
|
340
|
+
if @parent
|
341
|
+
if @join_field
|
342
|
+
join_value = event.get(@join_field)
|
343
|
+
parent_value = event.sprintf(@parent)
|
344
|
+
event.set(@join_field, { "name" => join_value, "parent" => parent_value })
|
345
|
+
params[routing_field_name] = event.sprintf(@parent)
|
346
|
+
else
|
347
|
+
params[:parent] = event.sprintf(@parent)
|
348
|
+
end
|
349
|
+
end
|
350
|
+
|
351
|
+
if action == 'update'
|
352
|
+
params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
|
353
|
+
params[:_script] = event.sprintf(@script) if @script != ""
|
354
|
+
params[retry_on_conflict_action_name] = @retry_on_conflict
|
355
|
+
end
|
356
|
+
|
357
|
+
if @version
|
358
|
+
params[:version] = event.sprintf(@version)
|
359
|
+
end
|
360
|
+
|
361
|
+
if @version_type
|
362
|
+
params[:version_type] = event.sprintf(@version_type)
|
363
|
+
end
|
364
|
+
|
365
|
+
[action, params, event]
|
366
|
+
end
|
367
|
+
|
368
|
+
# not private for elasticsearch_spec.rb
|
369
|
+
def retry_on_conflict_action_name
|
370
|
+
maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
|
318
371
|
end
|
319
372
|
|
320
373
|
@@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
|
@@ -324,4 +377,86 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
324
377
|
require "logstash/outputs/elasticsearch/#{name}"
|
325
378
|
end
|
326
379
|
|
327
|
-
|
380
|
+
private
|
381
|
+
|
382
|
+
def routing_field_name
|
383
|
+
maximum_seen_major_version >= 6 ? :routing : :_routing
|
384
|
+
end
|
385
|
+
|
386
|
+
# Determine the correct value for the 'type' field for the given event
|
387
|
+
DEFAULT_EVENT_TYPE_ES6="doc".freeze
|
388
|
+
DEFAULT_EVENT_TYPE_ES7="_doc".freeze
|
389
|
+
def get_event_type(event)
|
390
|
+
# Set the 'type' value for the index.
|
391
|
+
type = if @document_type
|
392
|
+
event.sprintf(@document_type)
|
393
|
+
else
|
394
|
+
if maximum_seen_major_version < 6
|
395
|
+
event.get("type") || DEFAULT_EVENT_TYPE_ES6
|
396
|
+
elsif maximum_seen_major_version == 6
|
397
|
+
DEFAULT_EVENT_TYPE_ES6
|
398
|
+
elsif maximum_seen_major_version == 7
|
399
|
+
DEFAULT_EVENT_TYPE_ES7
|
400
|
+
else
|
401
|
+
nil
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
if !(type.is_a?(String) || type.is_a?(Numeric))
|
406
|
+
@logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event)
|
407
|
+
end
|
408
|
+
|
409
|
+
type.to_s
|
410
|
+
end
|
411
|
+
|
412
|
+
##
|
413
|
+
# WARNING: This method is overridden in a subclass in Logstash Core 7.7-7.8's monitoring,
|
414
|
+
# where a `client` argument is both required and ignored. In later versions of
|
415
|
+
# Logstash Core it is optional and ignored, but to make it optional here would
|
416
|
+
# allow us to accidentally break compatibility with Logstashes where it was required.
|
417
|
+
# @param noop_required_client [nil]: required `nil` for legacy reasons.
|
418
|
+
# @return [Boolean]
|
419
|
+
def use_event_type?(noop_required_client)
|
420
|
+
maximum_seen_major_version < 8
|
421
|
+
end
|
422
|
+
|
423
|
+
def install_template
|
424
|
+
TemplateManager.install_template(self)
|
425
|
+
@template_installed.make_true
|
426
|
+
end
|
427
|
+
|
428
|
+
def setup_ecs_compatibility_related_defaults
|
429
|
+
case ecs_compatibility
|
430
|
+
when :disabled
|
431
|
+
@default_index = "logstash-%{+yyyy.MM.dd}"
|
432
|
+
@default_ilm_rollover_alias = "logstash"
|
433
|
+
@default_template_name = 'logstash'
|
434
|
+
when :v1
|
435
|
+
@default_index = "ecs-logstash-%{+yyyy.MM.dd}"
|
436
|
+
@default_ilm_rollover_alias = "ecs-logstash"
|
437
|
+
@default_template_name = 'ecs-logstash'
|
438
|
+
else
|
439
|
+
fail("unsupported ECS Compatibility `#{ecs_compatibility}`")
|
440
|
+
end
|
441
|
+
|
442
|
+
@index ||= default_index
|
443
|
+
@ilm_rollover_alias ||= default_ilm_rollover_alias
|
444
|
+
@template_name ||= default_template_name
|
445
|
+
end
|
446
|
+
|
447
|
+
# To be overidden by the -java version
|
448
|
+
VALID_HTTP_ACTIONS=["index", "delete", "create", "update"]
|
449
|
+
def valid_actions
|
450
|
+
VALID_HTTP_ACTIONS
|
451
|
+
end
|
452
|
+
|
453
|
+
def check_action_validity
|
454
|
+
raise LogStash::ConfigurationError, "No action specified!" unless @action
|
455
|
+
|
456
|
+
# If we're using string interpolation, we're good!
|
457
|
+
return if @action =~ /%{.+}/
|
458
|
+
return if valid_actions.include?(@action)
|
459
|
+
|
460
|
+
raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement"
|
461
|
+
end
|
462
|
+
end
|