logstash-output-elasticsearch-test 11.16.0-x86_64-linux
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +649 -0
- data/CONTRIBUTORS +34 -0
- data/Gemfile +16 -0
- data/LICENSE +202 -0
- data/NOTICE.TXT +5 -0
- data/README.md +106 -0
- data/docs/index.asciidoc +1369 -0
- data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +282 -0
- data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +155 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +534 -0
- data/lib/logstash/outputs/elasticsearch/http_client.rb +497 -0
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +201 -0
- data/lib/logstash/outputs/elasticsearch/ilm.rb +92 -0
- data/lib/logstash/outputs/elasticsearch/license_checker.rb +52 -0
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +131 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json +45 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json +44 -0
- data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json +50 -0
- data/lib/logstash/outputs/elasticsearch.rb +699 -0
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +237 -0
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +409 -0
- data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
- data/logstash-output-elasticsearch.gemspec +40 -0
- data/spec/es_spec_helper.rb +225 -0
- data/spec/fixtures/_nodes/6x.json +81 -0
- data/spec/fixtures/_nodes/7x.json +92 -0
- data/spec/fixtures/htpasswd +2 -0
- data/spec/fixtures/license_check/active.json +16 -0
- data/spec/fixtures/license_check/inactive.json +5 -0
- data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
- data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
- data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
- data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
- data/spec/fixtures/template-with-policy-es6x.json +48 -0
- data/spec/fixtures/template-with-policy-es7x.json +45 -0
- data/spec/fixtures/template-with-policy-es8x.json +50 -0
- data/spec/fixtures/test_certs/ca.crt +29 -0
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -0
- data/spec/fixtures/test_certs/ca.key +51 -0
- data/spec/fixtures/test_certs/renew.sh +13 -0
- data/spec/fixtures/test_certs/test.crt +30 -0
- data/spec/fixtures/test_certs/test.der.sha256 +1 -0
- data/spec/fixtures/test_certs/test.key +51 -0
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/fixtures/test_certs/test_invalid.crt +36 -0
- data/spec/fixtures/test_certs/test_invalid.key +51 -0
- data/spec/fixtures/test_certs/test_invalid.p12 +0 -0
- data/spec/fixtures/test_certs/test_self_signed.crt +32 -0
- data/spec/fixtures/test_certs/test_self_signed.key +54 -0
- data/spec/fixtures/test_certs/test_self_signed.p12 +0 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +70 -0
- data/spec/integration/outputs/create_spec.rb +67 -0
- data/spec/integration/outputs/data_stream_spec.rb +68 -0
- data/spec/integration/outputs/delete_spec.rb +63 -0
- data/spec/integration/outputs/ilm_spec.rb +534 -0
- data/spec/integration/outputs/index_spec.rb +421 -0
- data/spec/integration/outputs/index_version_spec.rb +98 -0
- data/spec/integration/outputs/ingest_pipeline_spec.rb +75 -0
- data/spec/integration/outputs/metrics_spec.rb +66 -0
- data/spec/integration/outputs/no_es_on_startup_spec.rb +78 -0
- data/spec/integration/outputs/painless_update_spec.rb +99 -0
- data/spec/integration/outputs/parent_spec.rb +94 -0
- data/spec/integration/outputs/retry_spec.rb +182 -0
- data/spec/integration/outputs/routing_spec.rb +61 -0
- data/spec/integration/outputs/sniffer_spec.rb +94 -0
- data/spec/integration/outputs/templates_spec.rb +133 -0
- data/spec/integration/outputs/unsupported_actions_spec.rb +75 -0
- data/spec/integration/outputs/update_spec.rb +114 -0
- data/spec/spec_helper.rb +10 -0
- data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
- data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
- data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
- data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
- data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
- data/spec/unit/http_client_builder_spec.rb +185 -0
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +612 -0
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +151 -0
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +501 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +339 -0
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +189 -0
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +103 -0
- data/spec/unit/outputs/elasticsearch_spec.rb +1573 -0
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +197 -0
- data/spec/unit/outputs/error_whitelist_spec.rb +56 -0
- data/spec/unit/outputs/license_check_spec.rb +57 -0
- metadata +423 -0
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,1369 @@
|
|
1
|
+
:plugin: elasticsearch
|
2
|
+
:type: output
|
3
|
+
:no_codec:
|
4
|
+
|
5
|
+
///////////////////////////////////////////
|
6
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
7
|
+
///////////////////////////////////////////
|
8
|
+
:version: %VERSION%
|
9
|
+
:release_date: %RELEASE_DATE%
|
10
|
+
:changelog_url: %CHANGELOG_URL%
|
11
|
+
:include_path: ../../../../logstash/docs/include
|
12
|
+
///////////////////////////////////////////
|
13
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
14
|
+
///////////////////////////////////////////
|
15
|
+
|
16
|
+
[id="plugins-{type}s-{plugin}"]
|
17
|
+
|
18
|
+
=== Elasticsearch output plugin
|
19
|
+
|
20
|
+
include::{include_path}/plugin_header.asciidoc[]
|
21
|
+
|
22
|
+
==== Description
|
23
|
+
|
24
|
+
Elasticsearch provides near real-time search and analytics for all types of
|
25
|
+
data. The Elasticsearch output plugin can store both time series datasets (such
|
26
|
+
as logs, events, and metrics) and non-time series data in Elasticsearch.
|
27
|
+
|
28
|
+
You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on
|
29
|
+
the website landing page or in the {ref}[Elasticsearch documentation].
|
30
|
+
|
31
|
+
.Compatibility Note
|
32
|
+
[NOTE]
|
33
|
+
================================================================================
|
34
|
+
When connected to Elasticsearch 7.x, modern versions of this plugin
|
35
|
+
don't use the document-type when inserting documents, unless the user
|
36
|
+
explicitly sets <<plugins-{type}s-{plugin}-document_type>>.
|
37
|
+
|
38
|
+
If you are using an earlier version of Logstash and wish to connect to
|
39
|
+
Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it
|
40
|
+
picks up changes to the Elasticsearch index template.
|
41
|
+
|
42
|
+
If you are using a custom <<plugins-{type}s-{plugin}-template>>,
|
43
|
+
ensure your template uses the `_doc` document-type before
|
44
|
+
connecting to Elasticsearch 7.x.
|
45
|
+
================================================================================
|
46
|
+
|
47
|
+
===== Hosted {es} Service on Elastic Cloud
|
48
|
+
|
49
|
+
{ess-leadin}
|
50
|
+
|
51
|
+
==== Compatibility with the Elastic Common Schema (ECS)
|
52
|
+
|
53
|
+
This plugin will persist events to Elasticsearch in the shape produced by
|
54
|
+
your pipeline, and _cannot_ be used to re-shape the event structure into a
|
55
|
+
shape that complies with ECS. To produce events that fully comply with ECS,
|
56
|
+
you will need to populate ECS-defined fields throughout your pipeline
|
57
|
+
definition.
|
58
|
+
|
59
|
+
However, the Elasticsearch Index Templates it manages can be configured to
|
60
|
+
be ECS-compatible by setting <<plugins-{type}s-{plugin}-ecs_compatibility>>.
|
61
|
+
By having an ECS-compatible template in place, we can ensure that Elasticsearch
|
62
|
+
is prepared to create and index fields in a way that is compatible with ECS,
|
63
|
+
and will correctly reject events with fields that conflict and cannot be coerced.
|
64
|
+
|
65
|
+
[id="plugins-{type}s-{plugin}-data-streams"]
|
66
|
+
==== Data streams
|
67
|
+
|
68
|
+
The {es} output plugin can store both time series datasets (such
|
69
|
+
as logs, events, and metrics) and non-time series data in Elasticsearch.
|
70
|
+
|
71
|
+
The data stream options are recommended for indexing time series datasets (such
|
72
|
+
as logs, metrics, and events) into {es}:
|
73
|
+
|
74
|
+
* <<plugins-{type}s-{plugin}-data_stream>>
|
75
|
+
* <<plugins-{type}s-{plugin}-data_stream_auto_routing>>
|
76
|
+
* <<plugins-{type}s-{plugin}-data_stream_dataset>>
|
77
|
+
* <<plugins-{type}s-{plugin}-data_stream_namespace>>
|
78
|
+
* <<plugins-{type}s-{plugin}-data_stream_sync_fields>>
|
79
|
+
* <<plugins-{type}s-{plugin}-data_stream_type>>
|
80
|
+
|
81
|
+
[id="plugins-{type}s-{plugin}-ds-examples"]
|
82
|
+
===== Data stream configuration examples
|
83
|
+
|
84
|
+
**Example: Basic default configuration**
|
85
|
+
|
86
|
+
[source,sh]
|
87
|
+
-----
|
88
|
+
output {
|
89
|
+
elasticsearch {
|
90
|
+
hosts => "hostname"
|
91
|
+
data_stream => "true"
|
92
|
+
}
|
93
|
+
}
|
94
|
+
-----
|
95
|
+
|
96
|
+
This example shows the minimal settings for processing data streams. Events
|
97
|
+
with `data_stream.*`` fields are routed to the appropriate data streams. If the
|
98
|
+
fields are missing, routing defaults to `logs-generic-logstash`.
|
99
|
+
|
100
|
+
**Example: Customize data stream name**
|
101
|
+
|
102
|
+
[source,sh]
|
103
|
+
-----
|
104
|
+
output {
|
105
|
+
elasticsearch {
|
106
|
+
hosts => "hostname"
|
107
|
+
data_stream => "true"
|
108
|
+
data_stream_type => "metrics"
|
109
|
+
data_stream_dataset => "foo"
|
110
|
+
data_stream_namespace => "bar"
|
111
|
+
}
|
112
|
+
}
|
113
|
+
-----
|
114
|
+
|
115
|
+
|
116
|
+
|
117
|
+
|
118
|
+
==== Writing to different indices: best practices
|
119
|
+
|
120
|
+
NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true`
|
121
|
+
and when using `ilm_rollover_alias`.
|
122
|
+
|
123
|
+
If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can:
|
124
|
+
|
125
|
+
* use different Elasticsearch outputs, each one with a different value for the `index` parameter
|
126
|
+
* use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter
|
127
|
+
|
128
|
+
Each Elasticsearch output is a new client connected to the cluster:
|
129
|
+
|
130
|
+
* it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients)
|
131
|
+
* it has an associated connection pool
|
132
|
+
|
133
|
+
In order to minimize the number of open connections to Elasticsearch, maximize
|
134
|
+
the bulk size and reduce the number of "small" bulk requests (which could easily
|
135
|
+
fill up the queue), it is usually more efficient to have a single Elasticsearch
|
136
|
+
output.
|
137
|
+
|
138
|
+
Example:
|
139
|
+
[source,ruby]
|
140
|
+
output {
|
141
|
+
elasticsearch {
|
142
|
+
index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}"
|
143
|
+
}
|
144
|
+
}
|
145
|
+
|
146
|
+
**What to do in case there is no field in the event containing the destination index prefix?**
|
147
|
+
|
148
|
+
You can use the `mutate` filter and conditionals to add a
|
149
|
+
{logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field]
|
150
|
+
to set the destination index for each event. The `[@metadata]` fields will not
|
151
|
+
be sent to Elasticsearch.
|
152
|
+
|
153
|
+
Example:
|
154
|
+
[source,ruby]
|
155
|
+
filter {
|
156
|
+
if [log_type] in [ "test", "staging" ] {
|
157
|
+
mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } }
|
158
|
+
} else if [log_type] == "production" {
|
159
|
+
mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } }
|
160
|
+
} else {
|
161
|
+
mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } }
|
162
|
+
}
|
163
|
+
}
|
164
|
+
output {
|
165
|
+
elasticsearch {
|
166
|
+
index => "%{[@metadata][target_index]}"
|
167
|
+
}
|
168
|
+
}
|
169
|
+
|
170
|
+
|
171
|
+
==== Retry Policy
|
172
|
+
|
173
|
+
The retry policy has changed significantly in the 8.1.1 release.
|
174
|
+
This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
|
175
|
+
either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP
|
176
|
+
request are handled differently than error codes for individual documents.
|
177
|
+
|
178
|
+
HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely.
|
179
|
+
|
180
|
+
The following document errors are handled as follows:
|
181
|
+
|
182
|
+
* 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<plugins-{type}s-{plugin}-dlq-policy>> for more info.
|
183
|
+
* 409 errors (conflict) are logged as a warning and dropped.
|
184
|
+
|
185
|
+
Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
|
186
|
+
It is more performant for Elasticsearch to retry these exceptions than this plugin.
|
187
|
+
|
188
|
+
[id="plugins-{type}s-{plugin}-dlq-policy"]
|
189
|
+
==== DLQ Policy
|
190
|
+
|
191
|
+
Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately
|
192
|
+
mapping errors cannot be handled without human intervention and without looking
|
193
|
+
at the field that caused the mapping mismatch. If the DLQ is enabled, the
|
194
|
+
original events causing the mapping errors are stored in a file that can be
|
195
|
+
processed at a later time. Often times, the offending field can be removed and
|
196
|
+
re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error
|
197
|
+
happens, the problem is logged as a warning, and the event is dropped. See
|
198
|
+
<<dead-letter-queues>> for more information about processing events in the DLQ.
|
199
|
+
The list of error codes accepted for DLQ could be customized with <<plugins-{type}s-{plugin}-dlq_custom_codes>>
|
200
|
+
but should be used only in motivated cases.
|
201
|
+
|
202
|
+
[id="plugins-{type}s-{plugin}-ilm"]
|
203
|
+
==== Index Lifecycle Management
|
204
|
+
|
205
|
+
[NOTE]
|
206
|
+
The Index Lifecycle Management feature requires plugin version `9.3.1` or higher.
|
207
|
+
|
208
|
+
[NOTE]
|
209
|
+
This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license
|
210
|
+
|
211
|
+
Logstash can use {ref}/index-lifecycle-management.html[Index Lifecycle
|
212
|
+
Management] to automate the management of indices over time.
|
213
|
+
|
214
|
+
The use of Index Lifecycle Management is controlled by the `ilm_enabled`
|
215
|
+
setting. By default, this setting detects whether the Elasticsearch instance
|
216
|
+
supports ILM, and uses it if it is available. `ilm_enabled` can also be set to
|
217
|
+
`true` or `false` to override the automatic detection, or disable ILM.
|
218
|
+
|
219
|
+
This will overwrite the index settings and adjust the Logstash template to write
|
220
|
+
the necessary settings for the template to support index lifecycle management,
|
221
|
+
including the index policy and rollover alias to be used.
|
222
|
+
|
223
|
+
Logstash will create a rollover alias for the indices to be written to,
|
224
|
+
including a pattern for how the actual indices will be named, and unless an ILM
|
225
|
+
policy that already exists has been specified, a default policy will also be
|
226
|
+
created. The default policy is configured to rollover an index when it reaches
|
227
|
+
either 50 gigabytes in size, or is 30 days old, whichever happens first.
|
228
|
+
|
229
|
+
The default rollover alias is called `logstash`, with a default pattern for the
|
230
|
+
rollover index of `{now/d}-00001`, which will name indices on the date that the
|
231
|
+
index is rolled over, followed by an incrementing number. Note that the pattern
|
232
|
+
must end with a dash and a number that will be incremented.
|
233
|
+
|
234
|
+
See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover
|
235
|
+
API documentation] for more details on naming.
|
236
|
+
|
237
|
+
The rollover alias, ilm pattern and policy can be modified.
|
238
|
+
|
239
|
+
See config below for an example:
|
240
|
+
[source,ruby]
|
241
|
+
output {
|
242
|
+
elasticsearch {
|
243
|
+
ilm_rollover_alias => "custom"
|
244
|
+
ilm_pattern => "000001"
|
245
|
+
ilm_policy => "custom_policy"
|
246
|
+
}
|
247
|
+
}
|
248
|
+
|
249
|
+
NOTE: Custom ILM policies must already exist on the Elasticsearch cluster before they can be used.
|
250
|
+
|
251
|
+
NOTE: If the rollover alias or pattern is modified, the index template will need to be
|
252
|
+
overwritten as the settings `index.lifecycle.name` and
|
253
|
+
`index.lifecycle.rollover_alias` are automatically written to the template
|
254
|
+
|
255
|
+
NOTE: If the index property is supplied in the output definition, it will be overwritten by the rollover alias.
|
256
|
+
|
257
|
+
|
258
|
+
==== Batch Sizes
|
259
|
+
|
260
|
+
This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es}
|
261
|
+
Bulk API] as a single request. However, if a batch exceeds 20MB we break it up
|
262
|
+
into multiple bulk requests. If a single document exceeds 20MB it is sent as a
|
263
|
+
single request.
|
264
|
+
|
265
|
+
==== DNS Caching
|
266
|
+
|
267
|
+
This plugin uses the JVM to lookup DNS entries and is subject to the value of
|
268
|
+
https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
|
269
|
+
a global setting for the JVM.
|
270
|
+
|
271
|
+
As an example, to set your DNS TTL to 1 second you would set
|
272
|
+
the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
|
273
|
+
|
274
|
+
Keep in mind that a connection with keepalive enabled will
|
275
|
+
not reevaluate its DNS value while the keepalive is in effect.
|
276
|
+
|
277
|
+
==== HTTP Compression
|
278
|
+
|
279
|
+
This plugin always reads compressed responses from {es}.
|
280
|
+
It _can be configured_ to send compressed bulk requests to {es}.
|
281
|
+
|
282
|
+
If you are concerned about bandwidth, you can enable <<plugins-{type}s-{plugin}-http_compression>> to trade a small amount of CPU capacity for a significant reduction in network IO.
|
283
|
+
|
284
|
+
==== Authentication
|
285
|
+
|
286
|
+
Authentication to a secure Elasticsearch cluster is possible using one of the
|
287
|
+
`user`/`password`, `cloud_auth` or `api_key` options.
|
288
|
+
|
289
|
+
[id="plugins-{type}s-{plugin}-autz"]
|
290
|
+
==== Authorization
|
291
|
+
|
292
|
+
Authorization to a secure Elasticsearch cluster requires `read` permission at
|
293
|
+
index level and `monitoring` permissions at cluster level. The `monitoring`
|
294
|
+
permission at cluster level is necessary to perform periodic connectivity
|
295
|
+
checks.
|
296
|
+
|
297
|
+
|
298
|
+
[id="plugins-{type}s-{plugin}-options"]
|
299
|
+
==== Elasticsearch Output Configuration Options
|
300
|
+
|
301
|
+
This plugin supports the following configuration options plus the
|
302
|
+
<<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
|
303
|
+
|
304
|
+
[cols="<,<,<",options="header",]
|
305
|
+
|=======================================================================
|
306
|
+
|Setting |Input type|Required
|
307
|
+
| <<plugins-{type}s-{plugin}-action>> |<<string,string>>|No
|
308
|
+
| <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
|
309
|
+
| <<plugins-{type}s-{plugin}-bulk_path>> |<<string,string>>|No
|
310
|
+
| <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
|
311
|
+
| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
|
312
|
+
| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
|
313
|
+
| <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
|
314
|
+
| <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>, one of `["true", "false", "auto"]`|No
|
315
|
+
| <<plugins-{type}s-{plugin}-data_stream_auto_routing>> |<<boolean,boolean>>|No
|
316
|
+
| <<plugins-{type}s-{plugin}-data_stream_dataset>> |<<string,string>>|No
|
317
|
+
| <<plugins-{type}s-{plugin}-data_stream_namespace>> |<<string,string>>|No
|
318
|
+
| <<plugins-{type}s-{plugin}-data_stream_sync_fields>> |<<boolean,boolean>>|No
|
319
|
+
| <<plugins-{type}s-{plugin}-data_stream_type>> |<<string,string>>|No
|
320
|
+
| <<plugins-{type}s-{plugin}-dlq_custom_codes>> |<<number,number>>|No
|
321
|
+
| <<plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |<<boolean,boolean>>|No
|
322
|
+
| <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
|
323
|
+
| <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
|
324
|
+
| <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
|
325
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
|
326
|
+
| <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |<<array,array>>|No
|
327
|
+
| <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
|
328
|
+
| <<plugins-{type}s-{plugin}-hosts>> |<<uri,uri>>|No
|
329
|
+
| <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
|
330
|
+
| <<plugins-{type}s-{plugin}-ilm_enabled>> |<<string,string>>, one of `["true", "false", "auto"]`|No
|
331
|
+
| <<plugins-{type}s-{plugin}-ilm_pattern>> |<<string,string>>|No
|
332
|
+
| <<plugins-{type}s-{plugin}-ilm_policy>> |<<string,string>>|No
|
333
|
+
| <<plugins-{type}s-{plugin}-ilm_rollover_alias>> |<<string,string>>|No
|
334
|
+
| <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
|
335
|
+
| <<plugins-{type}s-{plugin}-silence_errors_in_log>> |<<array,array>>|No
|
336
|
+
| <<plugins-{type}s-{plugin}-manage_template>> |<<boolean,boolean>>|No
|
337
|
+
| <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
|
338
|
+
| <<plugins-{type}s-{plugin}-parent>> |<<string,string>>|No
|
339
|
+
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
340
|
+
| <<plugins-{type}s-{plugin}-path>> |<<string,string>>|No
|
341
|
+
| <<plugins-{type}s-{plugin}-pipeline>> |<<string,string>>|No
|
342
|
+
| <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
|
343
|
+
| <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
|
344
|
+
| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
|
345
|
+
| <<plugins-{type}s-{plugin}-resurrect_delay>> |<<number,number>>|No
|
346
|
+
| <<plugins-{type}s-{plugin}-retry_initial_interval>> |<<number,number>>|No
|
347
|
+
| <<plugins-{type}s-{plugin}-retry_max_interval>> |<<number,number>>|No
|
348
|
+
| <<plugins-{type}s-{plugin}-retry_on_conflict>> |<<number,number>>|No
|
349
|
+
| <<plugins-{type}s-{plugin}-routing>> |<<string,string>>|No
|
350
|
+
| <<plugins-{type}s-{plugin}-script>> |<<string,string>>|No
|
351
|
+
| <<plugins-{type}s-{plugin}-script_lang>> |<<string,string>>|No
|
352
|
+
| <<plugins-{type}s-{plugin}-script_type>> |<<string,string>>, one of `["inline", "indexed", "file"]`|No
|
353
|
+
| <<plugins-{type}s-{plugin}-script_var_name>> |<<string,string>>|No
|
354
|
+
| <<plugins-{type}s-{plugin}-scripted_upsert>> |<<boolean,boolean>>|No
|
355
|
+
| <<plugins-{type}s-{plugin}-sniffing>> |<<boolean,boolean>>|No
|
356
|
+
| <<plugins-{type}s-{plugin}-sniffing_delay>> |<<number,number>>|No
|
357
|
+
| <<plugins-{type}s-{plugin}-sniffing_path>> |<<string,string>>|No
|
358
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
|
359
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
|
360
|
+
| <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of <<string,string>>|No
|
361
|
+
| <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
|
362
|
+
| <<plugins-{type}s-{plugin}-ssl_key>> |<<path,path>>|No
|
363
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
|
364
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_path>> |<<path,path>>|No
|
365
|
+
| <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
|
366
|
+
| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
|
367
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
|
368
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_path>> |<<path,path>>|No
|
369
|
+
| <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
|
370
|
+
| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>, one of `["full", "none"]`|No
|
371
|
+
| <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
|
372
|
+
| <<plugins-{type}s-{plugin}-template_api>> |<<string,string>>, one of `["auto", "legacy", "composable"]`|No
|
373
|
+
| <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
|
374
|
+
| <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
|
375
|
+
| <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
|
376
|
+
| <<plugins-{type}s-{plugin}-upsert>> |<<string,string>>|No
|
377
|
+
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
|
378
|
+
| <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
|
379
|
+
| <<plugins-{type}s-{plugin}-version>> |<<string,string>>|No
|
380
|
+
| <<plugins-{type}s-{plugin}-version_type>> |<<string,string>>, one of `["internal", "external", "external_gt", "external_gte", "force"]`|No
|
381
|
+
|=======================================================================
|
382
|
+
|
383
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
384
|
+
output plugins.
|
385
|
+
|
386
|
+
|
387
|
+
|
388
|
+
[id="plugins-{type}s-{plugin}-action"]
|
389
|
+
===== `action`
|
390
|
+
|
391
|
+
* Value type is <<string,string>>
|
392
|
+
* Default value is `create` for data streams, and `index` for non-time series data.
|
393
|
+
|
394
|
+
The Elasticsearch action to perform. Valid actions are:
|
395
|
+
|
396
|
+
- `index`: indexes a document (an event from Logstash).
|
397
|
+
- `delete`: deletes a document by id (An id is required for this action)
|
398
|
+
- `create`: indexes a document, fails if a document by that id already exists in the index.
|
399
|
+
- `update`: updates a document by id. Update has a special case where you can upsert -- update a
|
400
|
+
document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported
|
401
|
+
in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
|
402
|
+
- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
|
403
|
+
would use the foo field for the action.
|
404
|
+
If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}.
|
405
|
+
Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped.
|
406
|
+
|
407
|
+
For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation].
|
408
|
+
|
409
|
+
[id="plugins-{type}s-{plugin}-api_key"]
|
410
|
+
===== `api_key`
|
411
|
+
|
412
|
+
* Value type is <<password,password>>
|
413
|
+
* There is no default value for this setting.
|
414
|
+
|
415
|
+
Authenticate using Elasticsearch API key.
|
416
|
+
Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>.
|
417
|
+
|
418
|
+
Format is `id:api_key` where `id` and `api_key` are as returned by the
|
419
|
+
Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
|
420
|
+
|
421
|
+
[id="plugins-{type}s-{plugin}-bulk_path"]
|
422
|
+
===== `bulk_path`
|
423
|
+
|
424
|
+
* Value type is <<string,string>>
|
425
|
+
* There is no default value for this setting.
|
426
|
+
|
427
|
+
HTTP Path to perform the _bulk requests to
|
428
|
+
this defaults to a concatenation of the path parameter and "_bulk"
|
429
|
+
|
430
|
+
[id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
|
431
|
+
===== `ca_trusted_fingerprint`
|
432
|
+
|
433
|
+
* Value type is <<string,string>>, and must contain exactly 64 hexadecimal characters.
|
434
|
+
* There is no default value for this setting.
|
435
|
+
* Use of this option _requires_ Logstash 8.3+
|
436
|
+
|
437
|
+
The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster.
|
438
|
+
|
439
|
+
[id="plugins-{type}s-{plugin}-cloud_auth"]
|
440
|
+
===== `cloud_auth`
|
441
|
+
|
442
|
+
* Value type is <<password,password>>
|
443
|
+
* There is no default value for this setting.
|
444
|
+
|
445
|
+
Cloud authentication string ("<username>:<password>" format) is an alternative
|
446
|
+
for the `user`/`password` pair.
|
447
|
+
|
448
|
+
For more details, check out the
|
449
|
+
{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
|
450
|
+
|
451
|
+
[id="plugins-{type}s-{plugin}-cloud_id"]
|
452
|
+
===== `cloud_id`
|
453
|
+
|
454
|
+
* Value type is <<string,string>>
|
455
|
+
* There is no default value for this setting.
|
456
|
+
|
457
|
+
Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
|
458
|
+
|
459
|
+
For more details, check out the
|
460
|
+
{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
|
461
|
+
|
462
|
+
[id="plugins-{type}s-{plugin}-data_stream"]
|
463
|
+
===== `data_stream`
|
464
|
+
|
465
|
+
* Value can be any of: `true`, `false` and `auto`
|
466
|
+
* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
|
467
|
+
|
468
|
+
Defines whether data will be indexed into an Elasticsearch data stream.
|
469
|
+
The other `data_stream_*` settings will be used only if this setting is enabled.
|
470
|
+
|
471
|
+
Logstash handles the output as a data stream when the supplied configuration
|
472
|
+
is compatible with data streams and this value is set to `auto`.
|
473
|
+
|
474
|
+
[id="plugins-{type}s-{plugin}-data_stream_auto_routing"]
|
475
|
+
===== `data_stream_auto_routing`
|
476
|
+
|
477
|
+
* Value type is <<boolean,boolean>>
|
478
|
+
* Default value is `true`.
|
479
|
+
|
480
|
+
Automatically routes events by deriving the data stream name using specific event
|
481
|
+
fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format.
|
482
|
+
|
483
|
+
If enabled, the `data_stream.*` event fields will take precedence over the
|
484
|
+
`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings,
|
485
|
+
but will fall back to them if any of the fields are missing from the event.
|
486
|
+
|
487
|
+
[id="plugins-{type}s-{plugin}-data_stream_dataset"]
|
488
|
+
===== `data_stream_dataset`
|
489
|
+
|
490
|
+
* Value type is <<string,string>>
|
491
|
+
* Default value is `generic`.
|
492
|
+
|
493
|
+
The data stream dataset used to construct the data stream at index time.
|
494
|
+
|
495
|
+
[id="plugins-{type}s-{plugin}-data_stream_namespace"]
|
496
|
+
===== `data_stream_namespace`
|
497
|
+
|
498
|
+
* Value type is <<string,string>>
|
499
|
+
* Default value is `default`.
|
500
|
+
|
501
|
+
The data stream namespace used to construct the data stream at index time.
|
502
|
+
|
503
|
+
[id="plugins-{type}s-{plugin}-data_stream_sync_fields"]
|
504
|
+
===== `data_stream_sync_fields`
|
505
|
+
|
506
|
+
* Value type is <<boolean,boolean>>
|
507
|
+
* Default value is `true`
|
508
|
+
|
509
|
+
Automatically adds and syncs the `data_stream.*` event fields if they are missing from the
|
510
|
+
event. This ensures that fields match the name of the data stream that is receiving events.
|
511
|
+
|
512
|
+
NOTE: If existing `data_stream.*` event fields do not match the data stream name
|
513
|
+
and `data_stream_auto_routing` is disabled, the event fields will be
|
514
|
+
overwritten with a warning.
|
515
|
+
|
516
|
+
[id="plugins-{type}s-{plugin}-data_stream_type"]
|
517
|
+
===== `data_stream_type`
|
518
|
+
|
519
|
+
* Value type is <<string,string>>
|
520
|
+
* Default value is `logs`.
|
521
|
+
|
522
|
+
The data stream type used to construct the data stream at index time.
|
523
|
+
Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported.
|
524
|
+
|
525
|
+
[id="plugins-{type}s-{plugin}-dlq_custom_codes"]
|
526
|
+
===== `dlq_custom_codes`
|
527
|
+
|
528
|
+
* Value type is <<number,number>>
|
529
|
+
* Default value is `[]`.
|
530
|
+
|
531
|
+
List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue.
|
532
|
+
This list is an addition to the ordinary error codes considered for this feature, 400 and 404.
|
533
|
+
It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
|
534
|
+
The option accepts a list of natural numbers corresponding to HTTP errors codes.
|
535
|
+
|
536
|
+
[id="plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"]
|
537
|
+
===== `dlq_on_failed_indexname_interpolation`
|
538
|
+
|
539
|
+
* Value type is <<boolean,boolean>>
|
540
|
+
* Default value is `true`.
|
541
|
+
|
542
|
+
If enabled, failed index name interpolation events go into dead letter queue.
|
543
|
+
|
544
|
+
[id="plugins-{type}s-{plugin}-doc_as_upsert"]
|
545
|
+
===== `doc_as_upsert`
|
546
|
+
|
547
|
+
* Value type is <<boolean,boolean>>
|
548
|
+
* Default value is `false`
|
549
|
+
|
550
|
+
Enable `doc_as_upsert` for update mode.
|
551
|
+
Create a new document with source if `document_id` doesn't exist in Elasticsearch.
|
552
|
+
|
553
|
+
[id="plugins-{type}s-{plugin}-document_id"]
|
554
|
+
===== `document_id`
|
555
|
+
|
556
|
+
* Value type is <<string,string>>
|
557
|
+
* There is no default value for this setting.
|
558
|
+
|
559
|
+
The document ID for the index. Useful for overwriting existing entries in
|
560
|
+
Elasticsearch with the same ID.
|
561
|
+
|
562
|
+
[id="plugins-{type}s-{plugin}-document_type"]
|
563
|
+
===== `document_type`
|
564
|
+
|
565
|
+
* Value type is <<string,string>>
|
566
|
+
* There is no default value for this setting.
|
567
|
+
* This option is deprecated
|
568
|
+
|
569
|
+
NOTE: This option is deprecated due to the
|
570
|
+
https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal
|
571
|
+
of types in Elasticsearch 6.0]. It will be removed in the next major version of
|
572
|
+
Logstash.
|
573
|
+
|
574
|
+
NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`.
|
575
|
+
|
576
|
+
This sets the document type to write events to. Generally you should try to write only
|
577
|
+
similar events to the same 'type'. String expansion `%{foo}` works here.
|
578
|
+
If you don't set a value for this option:
|
579
|
+
|
580
|
+
- for elasticsearch clusters 8.x: no value will be used;
|
581
|
+
- for elasticsearch clusters 7.x: the value of '_doc' will be used;
|
582
|
+
- for elasticsearch clusters 6.x: the value of 'doc' will be used;
|
583
|
+
- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
|
584
|
+
|
585
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
586
|
+
===== `ecs_compatibility`
|
587
|
+
|
588
|
+
* Value type is <<string,string>>
|
589
|
+
* Supported values are:
|
590
|
+
** `disabled`: does not provide ECS-compatible templates
|
591
|
+
** `v1`,`v8`: Elastic Common Schema-compliant behavior
|
592
|
+
* Default value depends on which version of Logstash is running:
|
593
|
+
** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
|
594
|
+
** Otherwise, the default value is `disabled`.
|
595
|
+
|
596
|
+
Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema
|
597
|
+
(ECS)], including the installation of ECS-compatible index templates. The value
|
598
|
+
of this setting affects the _default_ values of:
|
599
|
+
|
600
|
+
* <<plugins-{type}s-{plugin}-index>>
|
601
|
+
* <<plugins-{type}s-{plugin}-template_name>>
|
602
|
+
* <<plugins-{type}s-{plugin}-ilm_rollover_alias>>
|
603
|
+
|
604
|
+
[id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]
|
605
|
+
===== `failure_type_logging_whitelist`
|
606
|
+
|
607
|
+
* Value type is <<array,array>>
|
608
|
+
* Default value is `[]`
|
609
|
+
|
610
|
+
NOTE: Deprecated, refer to <<plugins-{type}s-{plugin}-silence_errors_in_log>>.
|
611
|
+
|
612
|
+
[id="plugins-{type}s-{plugin}-custom_headers"]
|
613
|
+
===== `custom_headers`
|
614
|
+
|
615
|
+
* Value type is <<hash,hash>>
|
616
|
+
* There is no default value for this setting.
|
617
|
+
|
618
|
+
Pass a set of key value pairs as the headers sent in each request to
|
619
|
+
an elasticsearch node. The headers will be used for any kind of request
|
620
|
+
(_bulk request, template installation, health checks and sniffing).
|
621
|
+
These custom headers will be overidden by settings like `http_compression`.
|
622
|
+
|
623
|
+
[id="plugins-{type}s-{plugin}-healthcheck_path"]
|
624
|
+
===== `healthcheck_path`
|
625
|
+
|
626
|
+
* Value type is <<string,string>>
|
627
|
+
* There is no default value for this setting.
|
628
|
+
|
629
|
+
HTTP Path where a HEAD request is sent when a backend is marked down
|
630
|
+
the request is sent in the background to see if it has come back again
|
631
|
+
before it is once again eligible to service requests.
|
632
|
+
If you have custom firewall rules you may need to change this
|
633
|
+
|
634
|
+
[id="plugins-{type}s-{plugin}-hosts"]
|
635
|
+
===== `hosts`
|
636
|
+
|
637
|
+
* Value type is <<uri,uri>>
|
638
|
+
* Default value is `[//127.0.0.1]`
|
639
|
+
|
640
|
+
Sets the host(s) of the remote instance. If given an array it will load balance
|
641
|
+
requests across the hosts specified in the `hosts` parameter. Remember the
|
642
|
+
`http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg.
|
643
|
+
9200, not 9300).
|
644
|
+
|
645
|
+
Examples:
|
646
|
+
|
647
|
+
`"127.0.0.1"`
|
648
|
+
`["127.0.0.1:9200","127.0.0.2:9200"]`
|
649
|
+
`["http://127.0.0.1"]`
|
650
|
+
`["https://127.0.0.1:9200"]`
|
651
|
+
`["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
|
652
|
+
|
653
|
+
Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to
|
654
|
+
prevent Logstash from sending bulk requests to the master nodes. This parameter
|
655
|
+
should reference only data or client nodes in Elasticsearch.
|
656
|
+
|
657
|
+
Any special characters present in the URLs here MUST be URL escaped! This means
|
658
|
+
`#` should be put in as `%23` for instance.
|
659
|
+
|
660
|
+
[id="plugins-{type}s-{plugin}-http_compression"]
|
661
|
+
===== `http_compression`
|
662
|
+
|
663
|
+
* Value type is <<boolean,boolean>>
|
664
|
+
* Default value is `false`
|
665
|
+
|
666
|
+
Enable gzip compression on requests.
|
667
|
+
|
668
|
+
This setting allows you to reduce this plugin's outbound network traffic by
|
669
|
+
compressing each bulk _request_ to {es}.
|
670
|
+
|
671
|
+
NOTE: This output plugin reads compressed _responses_ from {es} regardless
|
672
|
+
of the value of this setting.
|
673
|
+
|
674
|
+
[id="plugins-{type}s-{plugin}-ilm_enabled"]
|
675
|
+
===== `ilm_enabled`
|
676
|
+
|
677
|
+
* Value can be any of: `true`, `false`, `auto`
|
678
|
+
* Default value is `auto`
|
679
|
+
|
680
|
+
The default setting of `auto` will automatically enable
|
681
|
+
{ref}/index-lifecycle-management.html[Index Lifecycle Management], if the
|
682
|
+
Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with
|
683
|
+
the ILM feature enabled, and disable it otherwise.
|
684
|
+
|
685
|
+
Setting this flag to `false` will disable the Index Lifecycle Management
|
686
|
+
feature, even if the Elasticsearch cluster supports ILM.
|
687
|
+
Setting this flag to `true` will enable Index Lifecycle Management feature, if
|
688
|
+
the Elasticsearch cluster supports it. This is required to enable Index
|
689
|
+
Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`.
|
690
|
+
|
691
|
+
NOTE: This feature requires a Basic License or above to be installed on an
|
692
|
+
Elasticsearch cluster version 6.6.0 or later.
|
693
|
+
|
694
|
+
[id="plugins-{type}s-{plugin}-ilm_pattern"]
|
695
|
+
===== `ilm_pattern`
|
696
|
+
|
697
|
+
* Value type is <<string,string>>
|
698
|
+
* Default value is `{now/d}-000001`
|
699
|
+
|
700
|
+
Pattern used for generating indices managed by
|
701
|
+
{ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value
|
702
|
+
specified in the pattern will be appended to the write alias, and incremented
|
703
|
+
automatically when a new index is created by ILM.
|
704
|
+
|
705
|
+
Date Math can be used when specifying an ilm pattern, see
|
706
|
+
{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover
|
707
|
+
API docs] for details.
|
708
|
+
|
709
|
+
NOTE: Updating the pattern will require the index template to be rewritten.
|
710
|
+
|
711
|
+
NOTE: The pattern must finish with a dash and a number that will be automatically
|
712
|
+
incremented when indices rollover.
|
713
|
+
|
714
|
+
NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name.
|
715
|
+
Example: 000001. See
|
716
|
+
{ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path
|
717
|
+
parameters API docs] for details.
|
718
|
+
|
719
|
+
[id="plugins-{type}s-{plugin}-ilm_policy"]
|
720
|
+
===== `ilm_policy`
|
721
|
+
|
722
|
+
* Value type is <<string,string>>
|
723
|
+
* Default value is `logstash-policy`
|
724
|
+
|
725
|
+
Modify this setting to use a custom Index Lifecycle Management policy, rather
|
726
|
+
than the default. If this value is not set, the default policy will be
|
727
|
+
automatically installed into Elasticsearch
|
728
|
+
|
729
|
+
NOTE: If this setting is specified, the policy must already exist in Elasticsearch
|
730
|
+
cluster.
|
731
|
+
|
732
|
+
[id="plugins-{type}s-{plugin}-ilm_rollover_alias"]
|
733
|
+
===== `ilm_rollover_alias`
|
734
|
+
|
735
|
+
* Value type is <<string,string>>
|
736
|
+
* Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
|
737
|
+
** ECS Compatibility disabled: `logstash`
|
738
|
+
** ECS Compatibility enabled: `ecs-logstash`
|
739
|
+
|
740
|
+
The rollover alias is the alias where indices managed using Index Lifecycle
|
741
|
+
Management will be written to.
|
742
|
+
|
743
|
+
NOTE: If both `index` and `ilm_rollover_alias` are specified,
|
744
|
+
`ilm_rollover_alias` takes precedence.
|
745
|
+
|
746
|
+
NOTE: Updating the rollover alias will require the index template to be
|
747
|
+
rewritten.
|
748
|
+
|
749
|
+
NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as
|
750
|
+
`index` does.
|
751
|
+
|
752
|
+
[id="plugins-{type}s-{plugin}-index"]
|
753
|
+
===== `index`
|
754
|
+
|
755
|
+
* Value type is <<string,string>>
|
756
|
+
* Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
|
757
|
+
** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"`
|
758
|
+
** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"`
|
759
|
+
|
760
|
+
The index to write events to. This can be dynamic using the `%{foo}` syntax.
|
761
|
+
The default value will partition your indices by day so you can more easily
|
762
|
+
delete old data or only search specific date ranges.
|
763
|
+
Indexes may not contain uppercase characters.
|
764
|
+
For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
|
765
|
+
Logstash uses
|
766
|
+
http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda
|
767
|
+
formats] and the `@timestamp` field of each event is being used as source for the date.
|
768
|
+
|
769
|
+
[id="plugins-{type}s-{plugin}-manage_template"]
|
770
|
+
===== `manage_template`
|
771
|
+
|
772
|
+
* Value type is <<boolean,boolean>>
|
773
|
+
* Default value is `true` for non-time series data, and `false` for data streams.
|
774
|
+
|
775
|
+
From Logstash 1.3 onwards, a template is applied to Elasticsearch during
|
776
|
+
Logstash's startup if one with the name <<plugins-{type}s-{plugin}-template_name>>
|
777
|
+
does not already exist.
|
778
|
+
By default, the contents of this template is the default template for
|
779
|
+
`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
|
780
|
+
`logstash-*`. Should you require support for other index names, or would like
|
781
|
+
to change the mappings in the template in general, a custom template can be
|
782
|
+
specified by setting `template` to the path of a template file.
|
783
|
+
|
784
|
+
Setting `manage_template` to false disables this feature. If you require more
|
785
|
+
control over template creation, (e.g. creating indices dynamically based on
|
786
|
+
field names) you should set `manage_template` to false and use the REST
|
787
|
+
API to apply your templates manually.
|
788
|
+
|
789
|
+
[id="plugins-{type}s-{plugin}-parameters"]
|
790
|
+
===== `parameters`
|
791
|
+
|
792
|
+
* Value type is <<hash,hash>>
|
793
|
+
* There is no default value for this setting.
|
794
|
+
|
795
|
+
Pass a set of key value pairs as the URL query string. This query string is added
|
796
|
+
to every host listed in the 'hosts' configuration. If the 'hosts' list contains
|
797
|
+
urls that already have query strings, the one specified here will be appended.
|
798
|
+
|
799
|
+
[id="plugins-{type}s-{plugin}-parent"]
|
800
|
+
===== `parent`
|
801
|
+
|
802
|
+
* Value type is <<string,string>>
|
803
|
+
* Default value is `nil`
|
804
|
+
|
805
|
+
For child documents, ID of the associated parent.
|
806
|
+
This can be dynamic using the `%{foo}` syntax.
|
807
|
+
|
808
|
+
[id="plugins-{type}s-{plugin}-password"]
|
809
|
+
===== `password`
|
810
|
+
|
811
|
+
* Value type is <<password,password>>
|
812
|
+
* There is no default value for this setting.
|
813
|
+
|
814
|
+
Password to authenticate to a secure Elasticsearch cluster
|
815
|
+
|
816
|
+
[id="plugins-{type}s-{plugin}-path"]
|
817
|
+
===== `path`
|
818
|
+
|
819
|
+
* Value type is <<string,string>>
|
820
|
+
* There is no default value for this setting.
|
821
|
+
|
822
|
+
HTTP Path at which the Elasticsearch server lives. Use this if you must run
|
823
|
+
Elasticsearch behind a proxy that remaps the root path for the Elasticsearch
|
824
|
+
HTTP API lives.
|
825
|
+
Note that if you use paths as components of URLs in the 'hosts' field you may
|
826
|
+
not also set this field. That will raise an error at startup
|
827
|
+
|
828
|
+
[id="plugins-{type}s-{plugin}-pipeline"]
|
829
|
+
===== `pipeline`
|
830
|
+
|
831
|
+
* Value type is <<string,string>>
|
832
|
+
* There is no default value.
|
833
|
+
|
834
|
+
Set which ingest pipeline you wish to execute for an event. You can also use
|
835
|
+
event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`.
|
836
|
+
The pipeline parameter won't be set if the value resolves to empty string ("").
|
837
|
+
|
838
|
+
[id="plugins-{type}s-{plugin}-pool_max"]
|
839
|
+
===== `pool_max`
|
840
|
+
|
841
|
+
* Value type is <<number,number>>
|
842
|
+
* Default value is `1000`
|
843
|
+
|
844
|
+
While the output tries to reuse connections efficiently we have a maximum.
|
845
|
+
This sets the maximum number of open connections the output will create.
|
846
|
+
Setting this too low may mean frequently closing / opening connections
|
847
|
+
which is bad.
|
848
|
+
|
849
|
+
[id="plugins-{type}s-{plugin}-pool_max_per_route"]
|
850
|
+
===== `pool_max_per_route`
|
851
|
+
|
852
|
+
* Value type is <<number,number>>
|
853
|
+
* Default value is `100`
|
854
|
+
|
855
|
+
While the output tries to reuse connections efficiently we have a maximum per endpoint.
|
856
|
+
This sets the maximum number of open connections per endpoint the output will create.
|
857
|
+
Setting this too low may mean frequently closing / opening connections
|
858
|
+
which is bad.
|
859
|
+
|
860
|
+
[id="plugins-{type}s-{plugin}-proxy"]
|
861
|
+
===== `proxy`
|
862
|
+
|
863
|
+
* Value type is <<uri,uri>>
|
864
|
+
* There is no default value for this setting.
|
865
|
+
|
866
|
+
Set the address of a forward HTTP proxy.
|
867
|
+
This setting accepts only URI arguments to prevent leaking credentials.
|
868
|
+
An empty string is treated as if proxy was not set. This is useful when using
|
869
|
+
environment variables e.g. `proxy => '${LS_PROXY:}'`.
|
870
|
+
|
871
|
+
[id="plugins-{type}s-{plugin}-resurrect_delay"]
|
872
|
+
===== `resurrect_delay`
|
873
|
+
|
874
|
+
* Value type is <<number,number>>
|
875
|
+
* Default value is `5`
|
876
|
+
|
877
|
+
How frequently, in seconds, to wait between resurrection attempts.
|
878
|
+
Resurrection is the process by which backend endpoints marked 'down' are checked
|
879
|
+
to see if they have come back to life
|
880
|
+
|
881
|
+
[id="plugins-{type}s-{plugin}-retry_initial_interval"]
|
882
|
+
===== `retry_initial_interval`
|
883
|
+
|
884
|
+
* Value type is <<number,number>>
|
885
|
+
* Default value is `2`
|
886
|
+
|
887
|
+
Set initial interval in seconds between bulk retries. Doubled on each retry up
|
888
|
+
to `retry_max_interval`
|
889
|
+
|
890
|
+
[id="plugins-{type}s-{plugin}-retry_max_interval"]
|
891
|
+
===== `retry_max_interval`
|
892
|
+
|
893
|
+
* Value type is <<number,number>>
|
894
|
+
* Default value is `64`
|
895
|
+
|
896
|
+
Set max interval in seconds between bulk retries.
|
897
|
+
|
898
|
+
[id="plugins-{type}s-{plugin}-retry_on_conflict"]
|
899
|
+
===== `retry_on_conflict`
|
900
|
+
|
901
|
+
* Value type is <<number,number>>
|
902
|
+
* Default value is `1`
|
903
|
+
|
904
|
+
The number of times Elasticsearch should internally retry an update/upserted document.
|
905
|
+
|
906
|
+
[id="plugins-{type}s-{plugin}-routing"]
|
907
|
+
===== `routing`
|
908
|
+
|
909
|
+
* Value type is <<string,string>>
|
910
|
+
* There is no default value for this setting.
|
911
|
+
|
912
|
+
A routing override to be applied to all processed events.
|
913
|
+
This can be dynamic using the `%{foo}` syntax.
|
914
|
+
|
915
|
+
[id="plugins-{type}s-{plugin}-script"]
|
916
|
+
===== `script`
|
917
|
+
|
918
|
+
* Value type is <<string,string>>
|
919
|
+
* Default value is `""`
|
920
|
+
|
921
|
+
Set script name for scripted update mode
|
922
|
+
|
923
|
+
Example:
|
924
|
+
[source,ruby]
|
925
|
+
output {
|
926
|
+
elasticsearch {
|
927
|
+
script => "ctx._source.message = params.event.get('message')"
|
928
|
+
}
|
929
|
+
}
|
930
|
+
|
931
|
+
[id="plugins-{type}s-{plugin}-script_lang"]
|
932
|
+
===== `script_lang`
|
933
|
+
|
934
|
+
* Value type is <<string,string>>
|
935
|
+
* Default value is `"painless"`
|
936
|
+
|
937
|
+
Set the language of the used script.
|
938
|
+
When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set
|
939
|
+
this parameter to `""` (empty string).
|
940
|
+
|
941
|
+
[id="plugins-{type}s-{plugin}-script_type"]
|
942
|
+
===== `script_type`
|
943
|
+
|
944
|
+
* Value can be any of: `inline`, `indexed`, `file`
|
945
|
+
* Default value is `["inline"]`
|
946
|
+
|
947
|
+
Define the type of script referenced by "script" variable
|
948
|
+
inline : "script" contains inline script
|
949
|
+
indexed : "script" contains the name of script directly indexed in elasticsearch
|
950
|
+
file : "script" contains the name of script stored in elasticsearch's config directory
|
951
|
+
|
952
|
+
[id="plugins-{type}s-{plugin}-script_var_name"]
|
953
|
+
===== `script_var_name`
|
954
|
+
|
955
|
+
* Value type is <<string,string>>
|
956
|
+
* Default value is `"event"`
|
957
|
+
|
958
|
+
Set variable name passed to script (scripted update)
|
959
|
+
|
960
|
+
[id="plugins-{type}s-{plugin}-scripted_upsert"]
|
961
|
+
===== `scripted_upsert`
|
962
|
+
|
963
|
+
* Value type is <<boolean,boolean>>
|
964
|
+
* Default value is `false`
|
965
|
+
|
966
|
+
if enabled, script is in charge of creating non-existent document (scripted update)
|
967
|
+
|
968
|
+
[id="plugins-{type}s-{plugin}-silence_errors_in_log"]
|
969
|
+
===== `silence_errors_in_log`
|
970
|
+
|
971
|
+
* Value type is <<array,array>>
|
972
|
+
* Default value is `[]`
|
973
|
+
|
974
|
+
Defines the list of Elasticsearch errors that you don't want to log.
|
975
|
+
A useful example is when you want to skip all 409 errors
|
976
|
+
which are `document_already_exists_exception`.
|
977
|
+
|
978
|
+
[source,ruby]
|
979
|
+
output {
|
980
|
+
elasticsearch {
|
981
|
+
silence_errors_in_log => ["document_already_exists_exception"]
|
982
|
+
}
|
983
|
+
}
|
984
|
+
|
985
|
+
NOTE: Deprecates <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>>.
|
986
|
+
|
987
|
+
[id="plugins-{type}s-{plugin}-sniffing"]
|
988
|
+
===== `sniffing`
|
989
|
+
|
990
|
+
* Value type is <<boolean,boolean>>
|
991
|
+
* Default value is `false`
|
992
|
+
|
993
|
+
This setting asks Elasticsearch for the list of all cluster nodes and adds them
|
994
|
+
to the hosts list.
|
995
|
+
For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will
|
996
|
+
be added to the hosts list, excluding master-only nodes.
|
997
|
+
|
998
|
+
[id="plugins-{type}s-{plugin}-sniffing_delay"]
|
999
|
+
===== `sniffing_delay`
|
1000
|
+
|
1001
|
+
* Value type is <<number,number>>
|
1002
|
+
* Default value is `5`
|
1003
|
+
|
1004
|
+
How long to wait, in seconds, between sniffing attempts
|
1005
|
+
|
1006
|
+
[id="plugins-{type}s-{plugin}-sniffing_path"]
|
1007
|
+
===== `sniffing_path`
|
1008
|
+
|
1009
|
+
* Value type is <<string,string>>
|
1010
|
+
* There is no default value for this setting.
|
1011
|
+
|
1012
|
+
HTTP Path to be used for the sniffing requests
|
1013
|
+
the default value is computed by concatenating the path value and "_nodes/http"
|
1014
|
+
if sniffing_path is set it will be used as an absolute path
|
1015
|
+
do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
|
1016
|
+
|
1017
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate"]
|
1018
|
+
===== `ssl_certificate`
|
1019
|
+
* Value type is <<path,path>>
|
1020
|
+
* There is no default value for this setting.
|
1021
|
+
|
1022
|
+
SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file.
|
1023
|
+
|
1024
|
+
NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_key>> is set.
|
1025
|
+
|
1026
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
|
1027
|
+
===== `ssl_certificate_authorities`
|
1028
|
+
|
1029
|
+
* Value type is a list of <<path,path>>
|
1030
|
+
* There is no default value for this setting
|
1031
|
+
|
1032
|
+
The .cer or .pem files to validate the server's certificate.
|
1033
|
+
|
1034
|
+
NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time.
|
1035
|
+
|
1036
|
+
[id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
|
1037
|
+
===== `ssl_cipher_suites`
|
1038
|
+
* Value type is a list of <<string,string>>
|
1039
|
+
* There is no default value for this setting
|
1040
|
+
|
1041
|
+
The list of cipher suites to use, listed by priorities.
|
1042
|
+
Supported cipher suites vary depending on the Java and protocol versions.
|
1043
|
+
|
1044
|
+
[id="plugins-{type}s-{plugin}-ssl_enabled"]
|
1045
|
+
===== `ssl_enabled`
|
1046
|
+
|
1047
|
+
* Value type is <<boolean,boolean>>
|
1048
|
+
* There is no default value for this setting.
|
1049
|
+
|
1050
|
+
Enable SSL/TLS secured communication to Elasticsearch cluster.
|
1051
|
+
Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
|
1052
|
+
If no explicit protocol is specified plain HTTP will be used.
|
1053
|
+
|
1054
|
+
[id="plugins-{type}s-{plugin}-ssl_key"]
|
1055
|
+
===== `ssl_key`
|
1056
|
+
* Value type is <<path,path>>
|
1057
|
+
* There is no default value for this setting.
|
1058
|
+
|
1059
|
+
OpenSSL-style RSA private key that corresponds to the <<plugins-{type}s-{plugin}-ssl_certificate>>.
|
1060
|
+
|
1061
|
+
NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_certificate>> is set.
|
1062
|
+
|
1063
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_password"]
|
1064
|
+
===== `ssl_keystore_password`
|
1065
|
+
|
1066
|
+
* Value type is <<password,password>>
|
1067
|
+
* There is no default value for this setting.
|
1068
|
+
|
1069
|
+
Set the keystore password
|
1070
|
+
|
1071
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_path"]
|
1072
|
+
===== `ssl_keystore_path`
|
1073
|
+
|
1074
|
+
* Value type is <<path,path>>
|
1075
|
+
* There is no default value for this setting.
|
1076
|
+
|
1077
|
+
The keystore used to present a certificate to the server.
|
1078
|
+
It can be either `.jks` or `.p12`
|
1079
|
+
|
1080
|
+
NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
|
1081
|
+
|
1082
|
+
[id="plugins-{type}s-{plugin}-ssl_keystore_type"]
|
1083
|
+
===== `ssl_keystore_type`
|
1084
|
+
|
1085
|
+
* Value can be any of: `jks`, `pkcs12`
|
1086
|
+
* If not provided, the value will be inferred from the keystore filename.
|
1087
|
+
|
1088
|
+
The format of the keystore file. It must be either `jks` or `pkcs12`.
|
1089
|
+
|
1090
|
+
[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
|
1091
|
+
===== `ssl_supported_protocols`
|
1092
|
+
|
1093
|
+
* Value type is <<string,string>>
|
1094
|
+
* Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
|
1095
|
+
* Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
|
1096
|
+
`'TLSv1.1'` is not considered secure and is only provided for legacy applications.
|
1097
|
+
|
1098
|
+
List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
|
1099
|
+
|
1100
|
+
For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the
|
1101
|
+
`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
|
1102
|
+
|
1103
|
+
NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
|
1104
|
+
the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
|
1105
|
+
the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
|
1106
|
+
|
1107
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_password"]
|
1108
|
+
===== `ssl_truststore_password`
|
1109
|
+
|
1110
|
+
* Value type is <<password,password>>
|
1111
|
+
* There is no default value for this setting.
|
1112
|
+
|
1113
|
+
Set the truststore password
|
1114
|
+
|
1115
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_path"]
|
1116
|
+
===== `ssl_truststore_path`
|
1117
|
+
|
1118
|
+
* Value type is <<path,path>>
|
1119
|
+
* There is no default value for this setting.
|
1120
|
+
|
1121
|
+
The truststore to validate the server's certificate.
|
1122
|
+
It can be either `.jks` or `.p12`.
|
1123
|
+
|
1124
|
+
NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time.
|
1125
|
+
|
1126
|
+
[id="plugins-{type}s-{plugin}-ssl_truststore_type"]
|
1127
|
+
===== `ssl_truststore_type`
|
1128
|
+
|
1129
|
+
* Value can be any of: `jks`, `pkcs12`
|
1130
|
+
* If not provided, the value will be inferred from the truststore filename.
|
1131
|
+
|
1132
|
+
The format of the truststore file. It must be either `jks` or `pkcs12`.
|
1133
|
+
|
1134
|
+
[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
|
1135
|
+
===== `ssl_verification_mode`
|
1136
|
+
|
1137
|
+
* Value can be any of: `full`, `none`
|
1138
|
+
* Default value is `full`
|
1139
|
+
|
1140
|
+
Defines how to verify the certificates presented by another party in the TLS connection:
|
1141
|
+
|
1142
|
+
`full` validates that the server certificate has an issue date that’s within
|
1143
|
+
the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and
|
1144
|
+
has a hostname or IP address that matches the names within the certificate.
|
1145
|
+
|
1146
|
+
`none` performs no certificate validation.
|
1147
|
+
|
1148
|
+
WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
|
1149
|
+
|
1150
|
+
[id="plugins-{type}s-{plugin}-template"]
|
1151
|
+
===== `template`
|
1152
|
+
|
1153
|
+
* Value type is <<path,path>>
|
1154
|
+
* There is no default value for this setting.
|
1155
|
+
|
1156
|
+
You can set the path to your own template here, if you so desire.
|
1157
|
+
If not set, the included template will be used.
|
1158
|
+
|
1159
|
+
[id="plugins-{type}s-{plugin}-template_api"]
|
1160
|
+
===== `template_api`
|
1161
|
+
|
1162
|
+
* Value can be any of: `auto`, `legacy`, `composable`
|
1163
|
+
* Default value is `auto`
|
1164
|
+
|
1165
|
+
The default setting of `auto` will use
|
1166
|
+
{ref}/index-templates.html[index template API] to create index template, if the
|
1167
|
+
Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher,
|
1168
|
+
and use {ref}/indices-templates-v1.html[legacy template API] otherwise.
|
1169
|
+
|
1170
|
+
Setting this flag to `legacy` will use legacy template API to create index template.
|
1171
|
+
Setting this flag to `composable` will use index template API to create index template.
|
1172
|
+
|
1173
|
+
NOTE: The format of template provided to <<plugins-{type}s-{plugin}-template>> needs to match the template API being used.
|
1174
|
+
|
1175
|
+
[id="plugins-{type}s-{plugin}-template_name"]
|
1176
|
+
===== `template_name`
|
1177
|
+
|
1178
|
+
* Value type is <<string,string>>
|
1179
|
+
* Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
|
1180
|
+
** ECS Compatibility disabled: `logstash`
|
1181
|
+
** ECS Compatibility enabled: `ecs-logstash`
|
1182
|
+
|
1183
|
+
|
1184
|
+
This configuration option defines how the template is named inside Elasticsearch.
|
1185
|
+
Note that if you have used the template management features and subsequently
|
1186
|
+
change this, you will need to prune the old template manually, e.g.
|
1187
|
+
|
1188
|
+
`curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
|
1189
|
+
|
1190
|
+
where `OldTemplateName` is whatever the former setting was.
|
1191
|
+
|
1192
|
+
[id="plugins-{type}s-{plugin}-template_overwrite"]
|
1193
|
+
===== `template_overwrite`
|
1194
|
+
|
1195
|
+
* Value type is <<boolean,boolean>>
|
1196
|
+
* Default value is `false`
|
1197
|
+
|
1198
|
+
The template_overwrite option will always overwrite the indicated template
|
1199
|
+
in Elasticsearch with either the one indicated by template or the included one.
|
1200
|
+
This option is set to false by default. If you always want to stay up to date
|
1201
|
+
with the template provided by Logstash, this option could be very useful to you.
|
1202
|
+
Likewise, if you have your own template file managed by puppet, for example, and
|
1203
|
+
you wanted to be able to update it regularly, this option could help there as well.
|
1204
|
+
|
1205
|
+
Please note that if you are using your own customized version of the Logstash
|
1206
|
+
template (logstash), setting this to true will make Logstash to overwrite
|
1207
|
+
the "logstash" template (i.e. removing all customized settings)
|
1208
|
+
|
1209
|
+
[id="plugins-{type}s-{plugin}-timeout"]
|
1210
|
+
===== `timeout`
|
1211
|
+
|
1212
|
+
* Value type is <<number,number>>
|
1213
|
+
* Default value is `60`
|
1214
|
+
|
1215
|
+
Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
|
1216
|
+
a timeout occurs, the request will be retried.
|
1217
|
+
|
1218
|
+
[id="plugins-{type}s-{plugin}-upsert"]
|
1219
|
+
===== `upsert`
|
1220
|
+
|
1221
|
+
* Value type is <<string,string>>
|
1222
|
+
* Default value is `""`
|
1223
|
+
|
1224
|
+
Set upsert content for update mode.
|
1225
|
+
Create a new document with this parameter as json string if `document_id` doesn't exists
|
1226
|
+
|
1227
|
+
[id="plugins-{type}s-{plugin}-user"]
|
1228
|
+
===== `user`
|
1229
|
+
|
1230
|
+
* Value type is <<string,string>>
|
1231
|
+
* There is no default value for this setting.
|
1232
|
+
|
1233
|
+
Username to authenticate to a secure Elasticsearch cluster
|
1234
|
+
|
1235
|
+
[id="plugins-{type}s-{plugin}-validate_after_inactivity"]
|
1236
|
+
===== `validate_after_inactivity`
|
1237
|
+
|
1238
|
+
* Value type is <<number,number>>
|
1239
|
+
* Default value is `10000`
|
1240
|
+
|
1241
|
+
How long to wait before checking for a stale connection to determine if a keepalive request is needed.
|
1242
|
+
Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly.
|
1243
|
+
|
1244
|
+
This client is based on Apache Commons. Here's how the
|
1245
|
+
https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache
|
1246
|
+
Commons documentation] describes this option: "Defines period of inactivity in
|
1247
|
+
milliseconds after which persistent connections must be re-validated prior to
|
1248
|
+
being leased to the consumer. Non-positive value passed to this method disables
|
1249
|
+
connection validation. This check helps detect connections that have become
|
1250
|
+
stale (half-closed) while kept inactive in the pool."
|
1251
|
+
|
1252
|
+
[id="plugins-{type}s-{plugin}-version"]
|
1253
|
+
===== `version`
|
1254
|
+
|
1255
|
+
* Value type is <<string,string>>
|
1256
|
+
* There is no default value for this setting.
|
1257
|
+
|
1258
|
+
The version to use for indexing. Use sprintf syntax like `%{my_version}` to use
|
1259
|
+
a field value here. See the
|
1260
|
+
https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support
|
1261
|
+
blog] for more information.
|
1262
|
+
|
1263
|
+
[id="plugins-{type}s-{plugin}-version_type"]
|
1264
|
+
===== `version_type`
|
1265
|
+
|
1266
|
+
* Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force`
|
1267
|
+
* There is no default value for this setting.
|
1268
|
+
|
1269
|
+
The version_type to use for indexing. See the
|
1270
|
+
https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support
|
1271
|
+
blog] and {ref}/docs-index_.html#_version_types[Version types] in the
|
1272
|
+
Elasticsearch documentation.
|
1273
|
+
|
1274
|
+
[id="plugins-{type}s-{plugin}-deprecated-options"]
|
1275
|
+
==== Elasticsearch Output Deprecated Configuration Options
|
1276
|
+
|
1277
|
+
This plugin supports the following deprecated configurations.
|
1278
|
+
|
1279
|
+
WARNING: Deprecated options are subject to removal in future releases.
|
1280
|
+
|
1281
|
+
[cols="<,<,<",options="header",]
|
1282
|
+
|=======================================================================
|
1283
|
+
|Setting|Input type|Replaced by
|
1284
|
+
| <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
|
1285
|
+
| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_keystore_path>>
|
1286
|
+
| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_keystore_password>>
|
1287
|
+
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_enabled>>
|
1288
|
+
| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_verification_mode>>
|
1289
|
+
| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_truststore_path>>
|
1290
|
+
| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_truststore_password>>
|
1291
|
+
|=======================================================================
|
1292
|
+
|
1293
|
+
|
1294
|
+
[id="plugins-{type}s-{plugin}-cacert"]
|
1295
|
+
===== `cacert`
|
1296
|
+
deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
|
1297
|
+
|
1298
|
+
* Value type is a list of <<path,path>>
|
1299
|
+
* There is no default value for this setting.
|
1300
|
+
|
1301
|
+
The .cer or .pem file to validate the server's certificate.
|
1302
|
+
|
1303
|
+
[id="plugins-{type}s-{plugin}-keystore"]
|
1304
|
+
===== `keystore`
|
1305
|
+
deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_path>>]
|
1306
|
+
|
1307
|
+
* Value type is <<path,path>>
|
1308
|
+
* There is no default value for this setting.
|
1309
|
+
|
1310
|
+
The keystore used to present a certificate to the server.
|
1311
|
+
It can be either .jks or .p12
|
1312
|
+
|
1313
|
+
NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
|
1314
|
+
|
1315
|
+
[id="plugins-{type}s-{plugin}-keystore_password"]
|
1316
|
+
===== `keystore_password`
|
1317
|
+
deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_password>>]
|
1318
|
+
|
1319
|
+
* Value type is <<password,password>>
|
1320
|
+
* There is no default value for this setting.
|
1321
|
+
|
1322
|
+
Set the keystore password
|
1323
|
+
|
1324
|
+
[id="plugins-{type}s-{plugin}-ssl"]
|
1325
|
+
===== `ssl`
|
1326
|
+
deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
|
1327
|
+
|
1328
|
+
* Value type is <<boolean,boolean>>
|
1329
|
+
* There is no default value for this setting.
|
1330
|
+
|
1331
|
+
Enable SSL/TLS secured communication to Elasticsearch cluster.
|
1332
|
+
Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
|
1333
|
+
If no explicit protocol is specified plain HTTP will be used.
|
1334
|
+
|
1335
|
+
[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
|
1336
|
+
===== `ssl_certificate_verification`
|
1337
|
+
deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_verification_mode>>]
|
1338
|
+
|
1339
|
+
* Value type is <<boolean,boolean>>
|
1340
|
+
* Default value is `true`
|
1341
|
+
|
1342
|
+
Option to validate the server's certificate. Disabling this severely compromises security.
|
1343
|
+
For more information on disabling certificate verification please read
|
1344
|
+
https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
|
1345
|
+
|
1346
|
+
[id="plugins-{type}s-{plugin}-truststore"]
|
1347
|
+
===== `truststore`
|
1348
|
+
deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_truststore_path>>]
|
1349
|
+
|
1350
|
+
* Value type is <<path,path>>
|
1351
|
+
* There is no default value for this setting.
|
1352
|
+
|
1353
|
+
The truststore to validate the server's certificate.
|
1354
|
+
It can be either `.jks` or `.p12`.
|
1355
|
+
Use either `:truststore` or `:cacert`.
|
1356
|
+
|
1357
|
+
[id="plugins-{type}s-{plugin}-truststore_password"]
|
1358
|
+
===== `truststore_password`
|
1359
|
+
deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_truststore_password>>]
|
1360
|
+
|
1361
|
+
* Value type is <<password,password>>
|
1362
|
+
* There is no default value for this setting.
|
1363
|
+
|
1364
|
+
Set the truststore password
|
1365
|
+
|
1366
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
1367
|
+
include::{include_path}/{type}.asciidoc[]
|
1368
|
+
|
1369
|
+
:no_codec!:
|