logstash-output-elasticsearch-test 11.16.0-x86_64-linux

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +649 -0
  3. data/CONTRIBUTORS +34 -0
  4. data/Gemfile +16 -0
  5. data/LICENSE +202 -0
  6. data/NOTICE.TXT +5 -0
  7. data/README.md +106 -0
  8. data/docs/index.asciidoc +1369 -0
  9. data/lib/logstash/outputs/elasticsearch/data_stream_support.rb +282 -0
  10. data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +14 -0
  11. data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +155 -0
  12. data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +534 -0
  13. data/lib/logstash/outputs/elasticsearch/http_client.rb +497 -0
  14. data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +201 -0
  15. data/lib/logstash/outputs/elasticsearch/ilm.rb +92 -0
  16. data/lib/logstash/outputs/elasticsearch/license_checker.rb +52 -0
  17. data/lib/logstash/outputs/elasticsearch/template_manager.rb +131 -0
  18. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json +45 -0
  19. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json +44 -0
  20. data/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json +50 -0
  21. data/lib/logstash/outputs/elasticsearch.rb +699 -0
  22. data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +237 -0
  23. data/lib/logstash/plugin_mixins/elasticsearch/common.rb +409 -0
  24. data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb +9 -0
  25. data/logstash-output-elasticsearch.gemspec +40 -0
  26. data/spec/es_spec_helper.rb +225 -0
  27. data/spec/fixtures/_nodes/6x.json +81 -0
  28. data/spec/fixtures/_nodes/7x.json +92 -0
  29. data/spec/fixtures/htpasswd +2 -0
  30. data/spec/fixtures/license_check/active.json +16 -0
  31. data/spec/fixtures/license_check/inactive.json +5 -0
  32. data/spec/fixtures/nginx_reverse_proxy.conf +22 -0
  33. data/spec/fixtures/scripts/painless/scripted_update.painless +2 -0
  34. data/spec/fixtures/scripts/painless/scripted_update_nested.painless +1 -0
  35. data/spec/fixtures/scripts/painless/scripted_upsert.painless +1 -0
  36. data/spec/fixtures/template-with-policy-es6x.json +48 -0
  37. data/spec/fixtures/template-with-policy-es7x.json +45 -0
  38. data/spec/fixtures/template-with-policy-es8x.json +50 -0
  39. data/spec/fixtures/test_certs/ca.crt +29 -0
  40. data/spec/fixtures/test_certs/ca.der.sha256 +1 -0
  41. data/spec/fixtures/test_certs/ca.key +51 -0
  42. data/spec/fixtures/test_certs/renew.sh +13 -0
  43. data/spec/fixtures/test_certs/test.crt +30 -0
  44. data/spec/fixtures/test_certs/test.der.sha256 +1 -0
  45. data/spec/fixtures/test_certs/test.key +51 -0
  46. data/spec/fixtures/test_certs/test.p12 +0 -0
  47. data/spec/fixtures/test_certs/test_invalid.crt +36 -0
  48. data/spec/fixtures/test_certs/test_invalid.key +51 -0
  49. data/spec/fixtures/test_certs/test_invalid.p12 +0 -0
  50. data/spec/fixtures/test_certs/test_self_signed.crt +32 -0
  51. data/spec/fixtures/test_certs/test_self_signed.key +54 -0
  52. data/spec/fixtures/test_certs/test_self_signed.p12 +0 -0
  53. data/spec/integration/outputs/compressed_indexing_spec.rb +70 -0
  54. data/spec/integration/outputs/create_spec.rb +67 -0
  55. data/spec/integration/outputs/data_stream_spec.rb +68 -0
  56. data/spec/integration/outputs/delete_spec.rb +63 -0
  57. data/spec/integration/outputs/ilm_spec.rb +534 -0
  58. data/spec/integration/outputs/index_spec.rb +421 -0
  59. data/spec/integration/outputs/index_version_spec.rb +98 -0
  60. data/spec/integration/outputs/ingest_pipeline_spec.rb +75 -0
  61. data/spec/integration/outputs/metrics_spec.rb +66 -0
  62. data/spec/integration/outputs/no_es_on_startup_spec.rb +78 -0
  63. data/spec/integration/outputs/painless_update_spec.rb +99 -0
  64. data/spec/integration/outputs/parent_spec.rb +94 -0
  65. data/spec/integration/outputs/retry_spec.rb +182 -0
  66. data/spec/integration/outputs/routing_spec.rb +61 -0
  67. data/spec/integration/outputs/sniffer_spec.rb +94 -0
  68. data/spec/integration/outputs/templates_spec.rb +133 -0
  69. data/spec/integration/outputs/unsupported_actions_spec.rb +75 -0
  70. data/spec/integration/outputs/update_spec.rb +114 -0
  71. data/spec/spec_helper.rb +10 -0
  72. data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb +19 -0
  73. data/spec/support/elasticsearch/api/actions/get_alias.rb +18 -0
  74. data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb +18 -0
  75. data/spec/support/elasticsearch/api/actions/put_alias.rb +24 -0
  76. data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb +25 -0
  77. data/spec/unit/http_client_builder_spec.rb +185 -0
  78. data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +612 -0
  79. data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +151 -0
  80. data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +501 -0
  81. data/spec/unit/outputs/elasticsearch/http_client_spec.rb +339 -0
  82. data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +189 -0
  83. data/spec/unit/outputs/elasticsearch_proxy_spec.rb +103 -0
  84. data/spec/unit/outputs/elasticsearch_spec.rb +1573 -0
  85. data/spec/unit/outputs/elasticsearch_ssl_spec.rb +197 -0
  86. data/spec/unit/outputs/error_whitelist_spec.rb +56 -0
  87. data/spec/unit/outputs/license_check_spec.rb +57 -0
  88. metadata +423 -0
@@ -0,0 +1,1369 @@
1
+ :plugin: elasticsearch
2
+ :type: output
3
+ :no_codec:
4
+
5
+ ///////////////////////////////////////////
6
+ START - GENERATED VARIABLES, DO NOT EDIT!
7
+ ///////////////////////////////////////////
8
+ :version: %VERSION%
9
+ :release_date: %RELEASE_DATE%
10
+ :changelog_url: %CHANGELOG_URL%
11
+ :include_path: ../../../../logstash/docs/include
12
+ ///////////////////////////////////////////
13
+ END - GENERATED VARIABLES, DO NOT EDIT!
14
+ ///////////////////////////////////////////
15
+
16
+ [id="plugins-{type}s-{plugin}"]
17
+
18
+ === Elasticsearch output plugin
19
+
20
+ include::{include_path}/plugin_header.asciidoc[]
21
+
22
+ ==== Description
23
+
24
+ Elasticsearch provides near real-time search and analytics for all types of
25
+ data. The Elasticsearch output plugin can store both time series datasets (such
26
+ as logs, events, and metrics) and non-time series data in Elasticsearch.
27
+
28
+ You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on
29
+ the website landing page or in the {ref}[Elasticsearch documentation].
30
+
31
+ .Compatibility Note
32
+ [NOTE]
33
+ ================================================================================
34
+ When connected to Elasticsearch 7.x, modern versions of this plugin
35
+ don't use the document-type when inserting documents, unless the user
36
+ explicitly sets <<plugins-{type}s-{plugin}-document_type>>.
37
+
38
+ If you are using an earlier version of Logstash and wish to connect to
39
+ Elasticsearch 7.x, first upgrade Logstash to version 6.8 to ensure it
40
+ picks up changes to the Elasticsearch index template.
41
+
42
+ If you are using a custom <<plugins-{type}s-{plugin}-template>>,
43
+ ensure your template uses the `_doc` document-type before
44
+ connecting to Elasticsearch 7.x.
45
+ ================================================================================
46
+
47
+ ===== Hosted {es} Service on Elastic Cloud
48
+
49
+ {ess-leadin}
50
+
51
+ ==== Compatibility with the Elastic Common Schema (ECS)
52
+
53
+ This plugin will persist events to Elasticsearch in the shape produced by
54
+ your pipeline, and _cannot_ be used to re-shape the event structure into a
55
+ shape that complies with ECS. To produce events that fully comply with ECS,
56
+ you will need to populate ECS-defined fields throughout your pipeline
57
+ definition.
58
+
59
+ However, the Elasticsearch Index Templates it manages can be configured to
60
+ be ECS-compatible by setting <<plugins-{type}s-{plugin}-ecs_compatibility>>.
61
+ By having an ECS-compatible template in place, we can ensure that Elasticsearch
62
+ is prepared to create and index fields in a way that is compatible with ECS,
63
+ and will correctly reject events with fields that conflict and cannot be coerced.
64
+
65
+ [id="plugins-{type}s-{plugin}-data-streams"]
66
+ ==== Data streams
67
+
68
+ The {es} output plugin can store both time series datasets (such
69
+ as logs, events, and metrics) and non-time series data in Elasticsearch.
70
+
71
+ The data stream options are recommended for indexing time series datasets (such
72
+ as logs, metrics, and events) into {es}:
73
+
74
+ * <<plugins-{type}s-{plugin}-data_stream>>
75
+ * <<plugins-{type}s-{plugin}-data_stream_auto_routing>>
76
+ * <<plugins-{type}s-{plugin}-data_stream_dataset>>
77
+ * <<plugins-{type}s-{plugin}-data_stream_namespace>>
78
+ * <<plugins-{type}s-{plugin}-data_stream_sync_fields>>
79
+ * <<plugins-{type}s-{plugin}-data_stream_type>>
80
+
81
+ [id="plugins-{type}s-{plugin}-ds-examples"]
82
+ ===== Data stream configuration examples
83
+
84
+ **Example: Basic default configuration**
85
+
86
+ [source,sh]
87
+ -----
88
+ output {
89
+ elasticsearch {
90
+ hosts => "hostname"
91
+ data_stream => "true"
92
+ }
93
+ }
94
+ -----
95
+
96
+ This example shows the minimal settings for processing data streams. Events
97
+ with `data_stream.*`` fields are routed to the appropriate data streams. If the
98
+ fields are missing, routing defaults to `logs-generic-logstash`.
99
+
100
+ **Example: Customize data stream name**
101
+
102
+ [source,sh]
103
+ -----
104
+ output {
105
+ elasticsearch {
106
+ hosts => "hostname"
107
+ data_stream => "true"
108
+ data_stream_type => "metrics"
109
+ data_stream_dataset => "foo"
110
+ data_stream_namespace => "bar"
111
+ }
112
+ }
113
+ -----
114
+
115
+
116
+
117
+
118
+ ==== Writing to different indices: best practices
119
+
120
+ NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true`
121
+ and when using `ilm_rollover_alias`.
122
+
123
+ If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can:
124
+
125
+ * use different Elasticsearch outputs, each one with a different value for the `index` parameter
126
+ * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter
127
+
128
+ Each Elasticsearch output is a new client connected to the cluster:
129
+
130
+ * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients)
131
+ * it has an associated connection pool
132
+
133
+ In order to minimize the number of open connections to Elasticsearch, maximize
134
+ the bulk size and reduce the number of "small" bulk requests (which could easily
135
+ fill up the queue), it is usually more efficient to have a single Elasticsearch
136
+ output.
137
+
138
+ Example:
139
+ [source,ruby]
140
+ output {
141
+ elasticsearch {
142
+ index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}"
143
+ }
144
+ }
145
+
146
+ **What to do in case there is no field in the event containing the destination index prefix?**
147
+
148
+ You can use the `mutate` filter and conditionals to add a
149
+ {logstash-ref}/event-dependent-configuration.html#metadata[`[@metadata]` field]
150
+ to set the destination index for each event. The `[@metadata]` fields will not
151
+ be sent to Elasticsearch.
152
+
153
+ Example:
154
+ [source,ruby]
155
+ filter {
156
+ if [log_type] in [ "test", "staging" ] {
157
+ mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } }
158
+ } else if [log_type] == "production" {
159
+ mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } }
160
+ } else {
161
+ mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } }
162
+ }
163
+ }
164
+ output {
165
+ elasticsearch {
166
+ index => "%{[@metadata][target_index]}"
167
+ }
168
+ }
169
+
170
+
171
+ ==== Retry Policy
172
+
173
+ The retry policy has changed significantly in the 8.1.1 release.
174
+ This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
175
+ either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP
176
+ request are handled differently than error codes for individual documents.
177
+
178
+ HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely.
179
+
180
+ The following document errors are handled as follows:
181
+
182
+ * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<plugins-{type}s-{plugin}-dlq-policy>> for more info.
183
+ * 409 errors (conflict) are logged as a warning and dropped.
184
+
185
+ Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
186
+ It is more performant for Elasticsearch to retry these exceptions than this plugin.
187
+
188
+ [id="plugins-{type}s-{plugin}-dlq-policy"]
189
+ ==== DLQ Policy
190
+
191
+ Mapping (404) errors from Elasticsearch can lead to data loss. Unfortunately
192
+ mapping errors cannot be handled without human intervention and without looking
193
+ at the field that caused the mapping mismatch. If the DLQ is enabled, the
194
+ original events causing the mapping errors are stored in a file that can be
195
+ processed at a later time. Often times, the offending field can be removed and
196
+ re-indexed to Elasticsearch. If the DLQ is not enabled, and a mapping error
197
+ happens, the problem is logged as a warning, and the event is dropped. See
198
+ <<dead-letter-queues>> for more information about processing events in the DLQ.
199
+ The list of error codes accepted for DLQ could be customized with <<plugins-{type}s-{plugin}-dlq_custom_codes>>
200
+ but should be used only in motivated cases.
201
+
202
+ [id="plugins-{type}s-{plugin}-ilm"]
203
+ ==== Index Lifecycle Management
204
+
205
+ [NOTE]
206
+ The Index Lifecycle Management feature requires plugin version `9.3.1` or higher.
207
+
208
+ [NOTE]
209
+ This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license
210
+
211
+ Logstash can use {ref}/index-lifecycle-management.html[Index Lifecycle
212
+ Management] to automate the management of indices over time.
213
+
214
+ The use of Index Lifecycle Management is controlled by the `ilm_enabled`
215
+ setting. By default, this setting detects whether the Elasticsearch instance
216
+ supports ILM, and uses it if it is available. `ilm_enabled` can also be set to
217
+ `true` or `false` to override the automatic detection, or disable ILM.
218
+
219
+ This will overwrite the index settings and adjust the Logstash template to write
220
+ the necessary settings for the template to support index lifecycle management,
221
+ including the index policy and rollover alias to be used.
222
+
223
+ Logstash will create a rollover alias for the indices to be written to,
224
+ including a pattern for how the actual indices will be named, and unless an ILM
225
+ policy that already exists has been specified, a default policy will also be
226
+ created. The default policy is configured to rollover an index when it reaches
227
+ either 50 gigabytes in size, or is 30 days old, whichever happens first.
228
+
229
+ The default rollover alias is called `logstash`, with a default pattern for the
230
+ rollover index of `{now/d}-00001`, which will name indices on the date that the
231
+ index is rolled over, followed by an incrementing number. Note that the pattern
232
+ must end with a dash and a number that will be incremented.
233
+
234
+ See the {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover
235
+ API documentation] for more details on naming.
236
+
237
+ The rollover alias, ilm pattern and policy can be modified.
238
+
239
+ See config below for an example:
240
+ [source,ruby]
241
+ output {
242
+ elasticsearch {
243
+ ilm_rollover_alias => "custom"
244
+ ilm_pattern => "000001"
245
+ ilm_policy => "custom_policy"
246
+ }
247
+ }
248
+
249
+ NOTE: Custom ILM policies must already exist on the Elasticsearch cluster before they can be used.
250
+
251
+ NOTE: If the rollover alias or pattern is modified, the index template will need to be
252
+ overwritten as the settings `index.lifecycle.name` and
253
+ `index.lifecycle.rollover_alias` are automatically written to the template
254
+
255
+ NOTE: If the index property is supplied in the output definition, it will be overwritten by the rollover alias.
256
+
257
+
258
+ ==== Batch Sizes
259
+
260
+ This plugin attempts to send batches of events to the {ref}/docs-bulk.html[{es}
261
+ Bulk API] as a single request. However, if a batch exceeds 20MB we break it up
262
+ into multiple bulk requests. If a single document exceeds 20MB it is sent as a
263
+ single request.
264
+
265
+ ==== DNS Caching
266
+
267
+ This plugin uses the JVM to lookup DNS entries and is subject to the value of
268
+ https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
269
+ a global setting for the JVM.
270
+
271
+ As an example, to set your DNS TTL to 1 second you would set
272
+ the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
273
+
274
+ Keep in mind that a connection with keepalive enabled will
275
+ not reevaluate its DNS value while the keepalive is in effect.
276
+
277
+ ==== HTTP Compression
278
+
279
+ This plugin always reads compressed responses from {es}.
280
+ It _can be configured_ to send compressed bulk requests to {es}.
281
+
282
+ If you are concerned about bandwidth, you can enable <<plugins-{type}s-{plugin}-http_compression>> to trade a small amount of CPU capacity for a significant reduction in network IO.
283
+
284
+ ==== Authentication
285
+
286
+ Authentication to a secure Elasticsearch cluster is possible using one of the
287
+ `user`/`password`, `cloud_auth` or `api_key` options.
288
+
289
+ [id="plugins-{type}s-{plugin}-autz"]
290
+ ==== Authorization
291
+
292
+ Authorization to a secure Elasticsearch cluster requires `read` permission at
293
+ index level and `monitoring` permissions at cluster level. The `monitoring`
294
+ permission at cluster level is necessary to perform periodic connectivity
295
+ checks.
296
+
297
+
298
+ [id="plugins-{type}s-{plugin}-options"]
299
+ ==== Elasticsearch Output Configuration Options
300
+
301
+ This plugin supports the following configuration options plus the
302
+ <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
303
+
304
+ [cols="<,<,<",options="header",]
305
+ |=======================================================================
306
+ |Setting |Input type|Required
307
+ | <<plugins-{type}s-{plugin}-action>> |<<string,string>>|No
308
+ | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
309
+ | <<plugins-{type}s-{plugin}-bulk_path>> |<<string,string>>|No
310
+ | <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
311
+ | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
312
+ | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
313
+ | <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
314
+ | <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>, one of `["true", "false", "auto"]`|No
315
+ | <<plugins-{type}s-{plugin}-data_stream_auto_routing>> |<<boolean,boolean>>|No
316
+ | <<plugins-{type}s-{plugin}-data_stream_dataset>> |<<string,string>>|No
317
+ | <<plugins-{type}s-{plugin}-data_stream_namespace>> |<<string,string>>|No
318
+ | <<plugins-{type}s-{plugin}-data_stream_sync_fields>> |<<boolean,boolean>>|No
319
+ | <<plugins-{type}s-{plugin}-data_stream_type>> |<<string,string>>|No
320
+ | <<plugins-{type}s-{plugin}-dlq_custom_codes>> |<<number,number>>|No
321
+ | <<plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |<<boolean,boolean>>|No
322
+ | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
323
+ | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
324
+ | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
325
+ | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
326
+ | <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |<<array,array>>|No
327
+ | <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
328
+ | <<plugins-{type}s-{plugin}-hosts>> |<<uri,uri>>|No
329
+ | <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
330
+ | <<plugins-{type}s-{plugin}-ilm_enabled>> |<<string,string>>, one of `["true", "false", "auto"]`|No
331
+ | <<plugins-{type}s-{plugin}-ilm_pattern>> |<<string,string>>|No
332
+ | <<plugins-{type}s-{plugin}-ilm_policy>> |<<string,string>>|No
333
+ | <<plugins-{type}s-{plugin}-ilm_rollover_alias>> |<<string,string>>|No
334
+ | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
335
+ | <<plugins-{type}s-{plugin}-silence_errors_in_log>> |<<array,array>>|No
336
+ | <<plugins-{type}s-{plugin}-manage_template>> |<<boolean,boolean>>|No
337
+ | <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
338
+ | <<plugins-{type}s-{plugin}-parent>> |<<string,string>>|No
339
+ | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
340
+ | <<plugins-{type}s-{plugin}-path>> |<<string,string>>|No
341
+ | <<plugins-{type}s-{plugin}-pipeline>> |<<string,string>>|No
342
+ | <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
343
+ | <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
344
+ | <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
345
+ | <<plugins-{type}s-{plugin}-resurrect_delay>> |<<number,number>>|No
346
+ | <<plugins-{type}s-{plugin}-retry_initial_interval>> |<<number,number>>|No
347
+ | <<plugins-{type}s-{plugin}-retry_max_interval>> |<<number,number>>|No
348
+ | <<plugins-{type}s-{plugin}-retry_on_conflict>> |<<number,number>>|No
349
+ | <<plugins-{type}s-{plugin}-routing>> |<<string,string>>|No
350
+ | <<plugins-{type}s-{plugin}-script>> |<<string,string>>|No
351
+ | <<plugins-{type}s-{plugin}-script_lang>> |<<string,string>>|No
352
+ | <<plugins-{type}s-{plugin}-script_type>> |<<string,string>>, one of `["inline", "indexed", "file"]`|No
353
+ | <<plugins-{type}s-{plugin}-script_var_name>> |<<string,string>>|No
354
+ | <<plugins-{type}s-{plugin}-scripted_upsert>> |<<boolean,boolean>>|No
355
+ | <<plugins-{type}s-{plugin}-sniffing>> |<<boolean,boolean>>|No
356
+ | <<plugins-{type}s-{plugin}-sniffing_delay>> |<<number,number>>|No
357
+ | <<plugins-{type}s-{plugin}-sniffing_path>> |<<string,string>>|No
358
+ | <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
359
+ | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
360
+ | <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of <<string,string>>|No
361
+ | <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
362
+ | <<plugins-{type}s-{plugin}-ssl_key>> |<<path,path>>|No
363
+ | <<plugins-{type}s-{plugin}-ssl_keystore_password>> |<<password,password>>|No
364
+ | <<plugins-{type}s-{plugin}-ssl_keystore_path>> |<<path,path>>|No
365
+ | <<plugins-{type}s-{plugin}-ssl_keystore_type>> |<<string,string>>|No
366
+ | <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
367
+ | <<plugins-{type}s-{plugin}-ssl_truststore_password>> |<<password,password>>|No
368
+ | <<plugins-{type}s-{plugin}-ssl_truststore_path>> |<<path,path>>|No
369
+ | <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
370
+ | <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>, one of `["full", "none"]`|No
371
+ | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
372
+ | <<plugins-{type}s-{plugin}-template_api>> |<<string,string>>, one of `["auto", "legacy", "composable"]`|No
373
+ | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
374
+ | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
375
+ | <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
376
+ | <<plugins-{type}s-{plugin}-upsert>> |<<string,string>>|No
377
+ | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
378
+ | <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
379
+ | <<plugins-{type}s-{plugin}-version>> |<<string,string>>|No
380
+ | <<plugins-{type}s-{plugin}-version_type>> |<<string,string>>, one of `["internal", "external", "external_gt", "external_gte", "force"]`|No
381
+ |=======================================================================
382
+
383
+ Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
384
+ output plugins.
385
+
386
+ &nbsp;
387
+
388
+ [id="plugins-{type}s-{plugin}-action"]
389
+ ===== `action`
390
+
391
+ * Value type is <<string,string>>
392
+ * Default value is `create` for data streams, and `index` for non-time series data.
393
+
394
+ The Elasticsearch action to perform. Valid actions are:
395
+
396
+ - `index`: indexes a document (an event from Logstash).
397
+ - `delete`: deletes a document by id (An id is required for this action)
398
+ - `create`: indexes a document, fails if a document by that id already exists in the index.
399
+ - `update`: updates a document by id. Update has a special case where you can upsert -- update a
400
+ document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported
401
+ in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
402
+ - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
403
+ would use the foo field for the action.
404
+ If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}.
405
+ Instead the event will be sent to the pipeline's {logstash-ref}/dead-letter-queues.html[dead-letter-queue (DLQ)] (if enabled), or it will be logged and dropped.
406
+
407
+ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation].
408
+
409
+ [id="plugins-{type}s-{plugin}-api_key"]
410
+ ===== `api_key`
411
+
412
+ * Value type is <<password,password>>
413
+ * There is no default value for this setting.
414
+
415
+ Authenticate using Elasticsearch API key.
416
+ Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>.
417
+
418
+ Format is `id:api_key` where `id` and `api_key` are as returned by the
419
+ Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
420
+
421
+ [id="plugins-{type}s-{plugin}-bulk_path"]
422
+ ===== `bulk_path`
423
+
424
+ * Value type is <<string,string>>
425
+ * There is no default value for this setting.
426
+
427
+ HTTP Path to perform the _bulk requests to
428
+ this defaults to a concatenation of the path parameter and "_bulk"
429
+
430
+ [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
431
+ ===== `ca_trusted_fingerprint`
432
+
433
+ * Value type is <<string,string>>, and must contain exactly 64 hexadecimal characters.
434
+ * There is no default value for this setting.
435
+ * Use of this option _requires_ Logstash 8.3+
436
+
437
+ The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster.
438
+
439
+ [id="plugins-{type}s-{plugin}-cloud_auth"]
440
+ ===== `cloud_auth`
441
+
442
+ * Value type is <<password,password>>
443
+ * There is no default value for this setting.
444
+
445
+ Cloud authentication string ("<username>:<password>" format) is an alternative
446
+ for the `user`/`password` pair.
447
+
448
+ For more details, check out the
449
+ {logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
450
+
451
+ [id="plugins-{type}s-{plugin}-cloud_id"]
452
+ ===== `cloud_id`
453
+
454
+ * Value type is <<string,string>>
455
+ * There is no default value for this setting.
456
+
457
+ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
458
+
459
+ For more details, check out the
460
+ {logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
461
+
462
+ [id="plugins-{type}s-{plugin}-data_stream"]
463
+ ===== `data_stream`
464
+
465
+ * Value can be any of: `true`, `false` and `auto`
466
+ * Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
467
+
468
+ Defines whether data will be indexed into an Elasticsearch data stream.
469
+ The other `data_stream_*` settings will be used only if this setting is enabled.
470
+
471
+ Logstash handles the output as a data stream when the supplied configuration
472
+ is compatible with data streams and this value is set to `auto`.
473
+
474
+ [id="plugins-{type}s-{plugin}-data_stream_auto_routing"]
475
+ ===== `data_stream_auto_routing`
476
+
477
+ * Value type is <<boolean,boolean>>
478
+ * Default value is `true`.
479
+
480
+ Automatically routes events by deriving the data stream name using specific event
481
+ fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format.
482
+
483
+ If enabled, the `data_stream.*` event fields will take precedence over the
484
+ `data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings,
485
+ but will fall back to them if any of the fields are missing from the event.
486
+
487
+ [id="plugins-{type}s-{plugin}-data_stream_dataset"]
488
+ ===== `data_stream_dataset`
489
+
490
+ * Value type is <<string,string>>
491
+ * Default value is `generic`.
492
+
493
+ The data stream dataset used to construct the data stream at index time.
494
+
495
+ [id="plugins-{type}s-{plugin}-data_stream_namespace"]
496
+ ===== `data_stream_namespace`
497
+
498
+ * Value type is <<string,string>>
499
+ * Default value is `default`.
500
+
501
+ The data stream namespace used to construct the data stream at index time.
502
+
503
+ [id="plugins-{type}s-{plugin}-data_stream_sync_fields"]
504
+ ===== `data_stream_sync_fields`
505
+
506
+ * Value type is <<boolean,boolean>>
507
+ * Default value is `true`
508
+
509
+ Automatically adds and syncs the `data_stream.*` event fields if they are missing from the
510
+ event. This ensures that fields match the name of the data stream that is receiving events.
511
+
512
+ NOTE: If existing `data_stream.*` event fields do not match the data stream name
513
+ and `data_stream_auto_routing` is disabled, the event fields will be
514
+ overwritten with a warning.
515
+
516
+ [id="plugins-{type}s-{plugin}-data_stream_type"]
517
+ ===== `data_stream_type`
518
+
519
+ * Value type is <<string,string>>
520
+ * Default value is `logs`.
521
+
522
+ The data stream type used to construct the data stream at index time.
523
+ Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported.
524
+
525
+ [id="plugins-{type}s-{plugin}-dlq_custom_codes"]
526
+ ===== `dlq_custom_codes`
527
+
528
+ * Value type is <<number,number>>
529
+ * Default value is `[]`.
530
+
531
+ List single-action error codes from Elasticsearch's Bulk API that are considered valid to move the events into the dead letter queue.
532
+ This list is an addition to the ordinary error codes considered for this feature, 400 and 404.
533
+ It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
534
+ The option accepts a list of natural numbers corresponding to HTTP errors codes.
535
+
536
+ [id="plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"]
537
+ ===== `dlq_on_failed_indexname_interpolation`
538
+
539
+ * Value type is <<boolean,boolean>>
540
+ * Default value is `true`.
541
+
542
+ If enabled, failed index name interpolation events go into dead letter queue.
543
+
544
+ [id="plugins-{type}s-{plugin}-doc_as_upsert"]
545
+ ===== `doc_as_upsert`
546
+
547
+ * Value type is <<boolean,boolean>>
548
+ * Default value is `false`
549
+
550
+ Enable `doc_as_upsert` for update mode.
551
+ Create a new document with source if `document_id` doesn't exist in Elasticsearch.
552
+
553
+ [id="plugins-{type}s-{plugin}-document_id"]
554
+ ===== `document_id`
555
+
556
+ * Value type is <<string,string>>
557
+ * There is no default value for this setting.
558
+
559
+ The document ID for the index. Useful for overwriting existing entries in
560
+ Elasticsearch with the same ID.
561
+
562
+ [id="plugins-{type}s-{plugin}-document_type"]
563
+ ===== `document_type`
564
+
565
+ * Value type is <<string,string>>
566
+ * There is no default value for this setting.
567
+ * This option is deprecated
568
+
569
+ NOTE: This option is deprecated due to the
570
+ https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal
571
+ of types in Elasticsearch 6.0]. It will be removed in the next major version of
572
+ Logstash.
573
+
574
+ NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`.
575
+
576
+ This sets the document type to write events to. Generally you should try to write only
577
+ similar events to the same 'type'. String expansion `%{foo}` works here.
578
+ If you don't set a value for this option:
579
+
580
+ - for elasticsearch clusters 8.x: no value will be used;
581
+ - for elasticsearch clusters 7.x: the value of '_doc' will be used;
582
+ - for elasticsearch clusters 6.x: the value of 'doc' will be used;
583
+ - for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
584
+
585
+ [id="plugins-{type}s-{plugin}-ecs_compatibility"]
586
+ ===== `ecs_compatibility`
587
+
588
+ * Value type is <<string,string>>
589
+ * Supported values are:
590
+ ** `disabled`: does not provide ECS-compatible templates
591
+ ** `v1`,`v8`: Elastic Common Schema-compliant behavior
592
+ * Default value depends on which version of Logstash is running:
593
+ ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
594
+ ** Otherwise, the default value is `disabled`.
595
+
596
+ Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema
597
+ (ECS)], including the installation of ECS-compatible index templates. The value
598
+ of this setting affects the _default_ values of:
599
+
600
+ * <<plugins-{type}s-{plugin}-index>>
601
+ * <<plugins-{type}s-{plugin}-template_name>>
602
+ * <<plugins-{type}s-{plugin}-ilm_rollover_alias>>
603
+
604
+ [id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]
605
+ ===== `failure_type_logging_whitelist`
606
+
607
+ * Value type is <<array,array>>
608
+ * Default value is `[]`
609
+
610
+ NOTE: Deprecated, refer to <<plugins-{type}s-{plugin}-silence_errors_in_log>>.
611
+
612
+ [id="plugins-{type}s-{plugin}-custom_headers"]
613
+ ===== `custom_headers`
614
+
615
+ * Value type is <<hash,hash>>
616
+ * There is no default value for this setting.
617
+
618
+ Pass a set of key value pairs as the headers sent in each request to
619
+ an elasticsearch node. The headers will be used for any kind of request
620
+ (_bulk request, template installation, health checks and sniffing).
621
+ These custom headers will be overidden by settings like `http_compression`.
622
+
623
+ [id="plugins-{type}s-{plugin}-healthcheck_path"]
624
+ ===== `healthcheck_path`
625
+
626
+ * Value type is <<string,string>>
627
+ * There is no default value for this setting.
628
+
629
+ HTTP Path where a HEAD request is sent when a backend is marked down
630
+ the request is sent in the background to see if it has come back again
631
+ before it is once again eligible to service requests.
632
+ If you have custom firewall rules you may need to change this
633
+
634
+ [id="plugins-{type}s-{plugin}-hosts"]
635
+ ===== `hosts`
636
+
637
+ * Value type is <<uri,uri>>
638
+ * Default value is `[//127.0.0.1]`
639
+
640
+ Sets the host(s) of the remote instance. If given an array it will load balance
641
+ requests across the hosts specified in the `hosts` parameter. Remember the
642
+ `http` protocol uses the {ref}/modules-http.html#modules-http[http] address (eg.
643
+ 9200, not 9300).
644
+
645
+ Examples:
646
+
647
+ `"127.0.0.1"`
648
+ `["127.0.0.1:9200","127.0.0.2:9200"]`
649
+ `["http://127.0.0.1"]`
650
+ `["https://127.0.0.1:9200"]`
651
+ `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
652
+
653
+ Exclude {ref}/modules-node.html[dedicated master nodes] from the `hosts` list to
654
+ prevent Logstash from sending bulk requests to the master nodes. This parameter
655
+ should reference only data or client nodes in Elasticsearch.
656
+
657
+ Any special characters present in the URLs here MUST be URL escaped! This means
658
+ `#` should be put in as `%23` for instance.
659
+
660
+ [id="plugins-{type}s-{plugin}-http_compression"]
661
+ ===== `http_compression`
662
+
663
+ * Value type is <<boolean,boolean>>
664
+ * Default value is `false`
665
+
666
+ Enable gzip compression on requests.
667
+
668
+ This setting allows you to reduce this plugin's outbound network traffic by
669
+ compressing each bulk _request_ to {es}.
670
+
671
+ NOTE: This output plugin reads compressed _responses_ from {es} regardless
672
+ of the value of this setting.
673
+
674
+ [id="plugins-{type}s-{plugin}-ilm_enabled"]
675
+ ===== `ilm_enabled`
676
+
677
+ * Value can be any of: `true`, `false`, `auto`
678
+ * Default value is `auto`
679
+
680
+ The default setting of `auto` will automatically enable
681
+ {ref}/index-lifecycle-management.html[Index Lifecycle Management], if the
682
+ Elasticsearch cluster is running Elasticsearch version `7.0.0` or higher with
683
+ the ILM feature enabled, and disable it otherwise.
684
+
685
+ Setting this flag to `false` will disable the Index Lifecycle Management
686
+ feature, even if the Elasticsearch cluster supports ILM.
687
+ Setting this flag to `true` will enable Index Lifecycle Management feature, if
688
+ the Elasticsearch cluster supports it. This is required to enable Index
689
+ Lifecycle Management on a version of Elasticsearch earlier than version `7.0.0`.
690
+
691
+ NOTE: This feature requires a Basic License or above to be installed on an
692
+ Elasticsearch cluster version 6.6.0 or later.
693
+
694
+ [id="plugins-{type}s-{plugin}-ilm_pattern"]
695
+ ===== `ilm_pattern`
696
+
697
+ * Value type is <<string,string>>
698
+ * Default value is `{now/d}-000001`
699
+
700
+ Pattern used for generating indices managed by
701
+ {ref}/index-lifecycle-management.html[Index Lifecycle Management]. The value
702
+ specified in the pattern will be appended to the write alias, and incremented
703
+ automatically when a new index is created by ILM.
704
+
705
+ Date Math can be used when specifying an ilm pattern, see
706
+ {ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover
707
+ API docs] for details.
708
+
709
+ NOTE: Updating the pattern will require the index template to be rewritten.
710
+
711
+ NOTE: The pattern must finish with a dash and a number that will be automatically
712
+ incremented when indices rollover.
713
+
714
+ NOTE: The pattern is a 6-digit string padded by zeros, regardless of prior index name.
715
+ Example: 000001. See
716
+ {ref}/indices-rollover-index.html#rollover-index-api-path-params[Rollover path
717
+ parameters API docs] for details.
718
+
719
+ [id="plugins-{type}s-{plugin}-ilm_policy"]
720
+ ===== `ilm_policy`
721
+
722
+ * Value type is <<string,string>>
723
+ * Default value is `logstash-policy`
724
+
725
+ Modify this setting to use a custom Index Lifecycle Management policy, rather
726
+ than the default. If this value is not set, the default policy will be
727
+ automatically installed into Elasticsearch
728
+
729
+ NOTE: If this setting is specified, the policy must already exist in Elasticsearch
730
+ cluster.
731
+
732
+ [id="plugins-{type}s-{plugin}-ilm_rollover_alias"]
733
+ ===== `ilm_rollover_alias`
734
+
735
+ * Value type is <<string,string>>
736
+ * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
737
+ ** ECS Compatibility disabled: `logstash`
738
+ ** ECS Compatibility enabled: `ecs-logstash`
739
+
740
+ The rollover alias is the alias where indices managed using Index Lifecycle
741
+ Management will be written to.
742
+
743
+ NOTE: If both `index` and `ilm_rollover_alias` are specified,
744
+ `ilm_rollover_alias` takes precedence.
745
+
746
+ NOTE: Updating the rollover alias will require the index template to be
747
+ rewritten.
748
+
749
+ NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as
750
+ `index` does.
751
+
752
+ [id="plugins-{type}s-{plugin}-index"]
753
+ ===== `index`
754
+
755
+ * Value type is <<string,string>>
756
+ * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
757
+ ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"`
758
+ ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"`
759
+
760
+ The index to write events to. This can be dynamic using the `%{foo}` syntax.
761
+ The default value will partition your indices by day so you can more easily
762
+ delete old data or only search specific date ranges.
763
+ Indexes may not contain uppercase characters.
764
+ For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
765
+ Logstash uses
766
+ http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda
767
+ formats] and the `@timestamp` field of each event is being used as source for the date.
768
+
769
+ [id="plugins-{type}s-{plugin}-manage_template"]
770
+ ===== `manage_template`
771
+
772
+ * Value type is <<boolean,boolean>>
773
+ * Default value is `true` for non-time series data, and `false` for data streams.
774
+
775
+ From Logstash 1.3 onwards, a template is applied to Elasticsearch during
776
+ Logstash's startup if one with the name <<plugins-{type}s-{plugin}-template_name>>
777
+ does not already exist.
778
+ By default, the contents of this template is the default template for
779
+ `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
780
+ `logstash-*`. Should you require support for other index names, or would like
781
+ to change the mappings in the template in general, a custom template can be
782
+ specified by setting `template` to the path of a template file.
783
+
784
+ Setting `manage_template` to false disables this feature. If you require more
785
+ control over template creation, (e.g. creating indices dynamically based on
786
+ field names) you should set `manage_template` to false and use the REST
787
+ API to apply your templates manually.
788
+
789
+ [id="plugins-{type}s-{plugin}-parameters"]
790
+ ===== `parameters`
791
+
792
+ * Value type is <<hash,hash>>
793
+ * There is no default value for this setting.
794
+
795
+ Pass a set of key value pairs as the URL query string. This query string is added
796
+ to every host listed in the 'hosts' configuration. If the 'hosts' list contains
797
+ urls that already have query strings, the one specified here will be appended.
798
+
799
+ [id="plugins-{type}s-{plugin}-parent"]
800
+ ===== `parent`
801
+
802
+ * Value type is <<string,string>>
803
+ * Default value is `nil`
804
+
805
+ For child documents, ID of the associated parent.
806
+ This can be dynamic using the `%{foo}` syntax.
807
+
808
+ [id="plugins-{type}s-{plugin}-password"]
809
+ ===== `password`
810
+
811
+ * Value type is <<password,password>>
812
+ * There is no default value for this setting.
813
+
814
+ Password to authenticate to a secure Elasticsearch cluster
815
+
816
+ [id="plugins-{type}s-{plugin}-path"]
817
+ ===== `path`
818
+
819
+ * Value type is <<string,string>>
820
+ * There is no default value for this setting.
821
+
822
+ HTTP Path at which the Elasticsearch server lives. Use this if you must run
823
+ Elasticsearch behind a proxy that remaps the root path for the Elasticsearch
824
+ HTTP API lives.
825
+ Note that if you use paths as components of URLs in the 'hosts' field you may
826
+ not also set this field. That will raise an error at startup
827
+
828
+ [id="plugins-{type}s-{plugin}-pipeline"]
829
+ ===== `pipeline`
830
+
831
+ * Value type is <<string,string>>
832
+ * There is no default value.
833
+
834
+ Set which ingest pipeline you wish to execute for an event. You can also use
835
+ event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`.
836
+ The pipeline parameter won't be set if the value resolves to empty string ("").
837
+
838
+ [id="plugins-{type}s-{plugin}-pool_max"]
839
+ ===== `pool_max`
840
+
841
+ * Value type is <<number,number>>
842
+ * Default value is `1000`
843
+
844
+ While the output tries to reuse connections efficiently we have a maximum.
845
+ This sets the maximum number of open connections the output will create.
846
+ Setting this too low may mean frequently closing / opening connections
847
+ which is bad.
848
+
849
+ [id="plugins-{type}s-{plugin}-pool_max_per_route"]
850
+ ===== `pool_max_per_route`
851
+
852
+ * Value type is <<number,number>>
853
+ * Default value is `100`
854
+
855
+ While the output tries to reuse connections efficiently we have a maximum per endpoint.
856
+ This sets the maximum number of open connections per endpoint the output will create.
857
+ Setting this too low may mean frequently closing / opening connections
858
+ which is bad.
859
+
860
+ [id="plugins-{type}s-{plugin}-proxy"]
861
+ ===== `proxy`
862
+
863
+ * Value type is <<uri,uri>>
864
+ * There is no default value for this setting.
865
+
866
+ Set the address of a forward HTTP proxy.
867
+ This setting accepts only URI arguments to prevent leaking credentials.
868
+ An empty string is treated as if proxy was not set. This is useful when using
869
+ environment variables e.g. `proxy => '${LS_PROXY:}'`.
870
+
871
+ [id="plugins-{type}s-{plugin}-resurrect_delay"]
872
+ ===== `resurrect_delay`
873
+
874
+ * Value type is <<number,number>>
875
+ * Default value is `5`
876
+
877
+ How frequently, in seconds, to wait between resurrection attempts.
878
+ Resurrection is the process by which backend endpoints marked 'down' are checked
879
+ to see if they have come back to life
880
+
881
+ [id="plugins-{type}s-{plugin}-retry_initial_interval"]
882
+ ===== `retry_initial_interval`
883
+
884
+ * Value type is <<number,number>>
885
+ * Default value is `2`
886
+
887
+ Set initial interval in seconds between bulk retries. Doubled on each retry up
888
+ to `retry_max_interval`
889
+
890
+ [id="plugins-{type}s-{plugin}-retry_max_interval"]
891
+ ===== `retry_max_interval`
892
+
893
+ * Value type is <<number,number>>
894
+ * Default value is `64`
895
+
896
+ Set max interval in seconds between bulk retries.
897
+
898
+ [id="plugins-{type}s-{plugin}-retry_on_conflict"]
899
+ ===== `retry_on_conflict`
900
+
901
+ * Value type is <<number,number>>
902
+ * Default value is `1`
903
+
904
+ The number of times Elasticsearch should internally retry an update/upserted document.
905
+
906
+ [id="plugins-{type}s-{plugin}-routing"]
907
+ ===== `routing`
908
+
909
+ * Value type is <<string,string>>
910
+ * There is no default value for this setting.
911
+
912
+ A routing override to be applied to all processed events.
913
+ This can be dynamic using the `%{foo}` syntax.
914
+
915
+ [id="plugins-{type}s-{plugin}-script"]
916
+ ===== `script`
917
+
918
+ * Value type is <<string,string>>
919
+ * Default value is `""`
920
+
921
+ Set script name for scripted update mode
922
+
923
+ Example:
924
+ [source,ruby]
925
+ output {
926
+ elasticsearch {
927
+ script => "ctx._source.message = params.event.get('message')"
928
+ }
929
+ }
930
+
931
+ [id="plugins-{type}s-{plugin}-script_lang"]
932
+ ===== `script_lang`
933
+
934
+ * Value type is <<string,string>>
935
+ * Default value is `"painless"`
936
+
937
+ Set the language of the used script.
938
+ When using indexed (stored) scripts on Elasticsearch 6.0 and higher, you must set
939
+ this parameter to `""` (empty string).
940
+
941
+ [id="plugins-{type}s-{plugin}-script_type"]
942
+ ===== `script_type`
943
+
944
+ * Value can be any of: `inline`, `indexed`, `file`
945
+ * Default value is `["inline"]`
946
+
947
+ Define the type of script referenced by "script" variable
948
+ inline : "script" contains inline script
949
+ indexed : "script" contains the name of script directly indexed in elasticsearch
950
+ file : "script" contains the name of script stored in elasticsearch's config directory
951
+
952
+ [id="plugins-{type}s-{plugin}-script_var_name"]
953
+ ===== `script_var_name`
954
+
955
+ * Value type is <<string,string>>
956
+ * Default value is `"event"`
957
+
958
+ Set variable name passed to script (scripted update)
959
+
960
+ [id="plugins-{type}s-{plugin}-scripted_upsert"]
961
+ ===== `scripted_upsert`
962
+
963
+ * Value type is <<boolean,boolean>>
964
+ * Default value is `false`
965
+
966
+ if enabled, script is in charge of creating non-existent document (scripted update)
967
+
968
+ [id="plugins-{type}s-{plugin}-silence_errors_in_log"]
969
+ ===== `silence_errors_in_log`
970
+
971
+ * Value type is <<array,array>>
972
+ * Default value is `[]`
973
+
974
+ Defines the list of Elasticsearch errors that you don't want to log.
975
+ A useful example is when you want to skip all 409 errors
976
+ which are `document_already_exists_exception`.
977
+
978
+ [source,ruby]
979
+ output {
980
+ elasticsearch {
981
+ silence_errors_in_log => ["document_already_exists_exception"]
982
+ }
983
+ }
984
+
985
+ NOTE: Deprecates <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>>.
986
+
987
+ [id="plugins-{type}s-{plugin}-sniffing"]
988
+ ===== `sniffing`
989
+
990
+ * Value type is <<boolean,boolean>>
991
+ * Default value is `false`
992
+
993
+ This setting asks Elasticsearch for the list of all cluster nodes and adds them
994
+ to the hosts list.
995
+ For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will
996
+ be added to the hosts list, excluding master-only nodes.
997
+
998
+ [id="plugins-{type}s-{plugin}-sniffing_delay"]
999
+ ===== `sniffing_delay`
1000
+
1001
+ * Value type is <<number,number>>
1002
+ * Default value is `5`
1003
+
1004
+ How long to wait, in seconds, between sniffing attempts
1005
+
1006
+ [id="plugins-{type}s-{plugin}-sniffing_path"]
1007
+ ===== `sniffing_path`
1008
+
1009
+ * Value type is <<string,string>>
1010
+ * There is no default value for this setting.
1011
+
1012
+ HTTP Path to be used for the sniffing requests
1013
+ the default value is computed by concatenating the path value and "_nodes/http"
1014
+ if sniffing_path is set it will be used as an absolute path
1015
+ do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
1016
+
1017
+ [id="plugins-{type}s-{plugin}-ssl_certificate"]
1018
+ ===== `ssl_certificate`
1019
+ * Value type is <<path,path>>
1020
+ * There is no default value for this setting.
1021
+
1022
+ SSL certificate to use to authenticate the client. This certificate should be an OpenSSL-style X.509 certificate file.
1023
+
1024
+ NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_key>> is set.
1025
+
1026
+ [id="plugins-{type}s-{plugin}-ssl_certificate_authorities"]
1027
+ ===== `ssl_certificate_authorities`
1028
+
1029
+ * Value type is a list of <<path,path>>
1030
+ * There is no default value for this setting
1031
+
1032
+ The .cer or .pem files to validate the server's certificate.
1033
+
1034
+ NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_truststore_path>> at the same time.
1035
+
1036
+ [id="plugins-{type}s-{plugin}-ssl_cipher_suites"]
1037
+ ===== `ssl_cipher_suites`
1038
+ * Value type is a list of <<string,string>>
1039
+ * There is no default value for this setting
1040
+
1041
+ The list of cipher suites to use, listed by priorities.
1042
+ Supported cipher suites vary depending on the Java and protocol versions.
1043
+
1044
+ [id="plugins-{type}s-{plugin}-ssl_enabled"]
1045
+ ===== `ssl_enabled`
1046
+
1047
+ * Value type is <<boolean,boolean>>
1048
+ * There is no default value for this setting.
1049
+
1050
+ Enable SSL/TLS secured communication to Elasticsearch cluster.
1051
+ Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
1052
+ If no explicit protocol is specified plain HTTP will be used.
1053
+
1054
+ [id="plugins-{type}s-{plugin}-ssl_key"]
1055
+ ===== `ssl_key`
1056
+ * Value type is <<path,path>>
1057
+ * There is no default value for this setting.
1058
+
1059
+ OpenSSL-style RSA private key that corresponds to the <<plugins-{type}s-{plugin}-ssl_certificate>>.
1060
+
1061
+ NOTE: This setting can be used only if <<plugins-{type}s-{plugin}-ssl_certificate>> is set.
1062
+
1063
+ [id="plugins-{type}s-{plugin}-ssl_keystore_password"]
1064
+ ===== `ssl_keystore_password`
1065
+
1066
+ * Value type is <<password,password>>
1067
+ * There is no default value for this setting.
1068
+
1069
+ Set the keystore password
1070
+
1071
+ [id="plugins-{type}s-{plugin}-ssl_keystore_path"]
1072
+ ===== `ssl_keystore_path`
1073
+
1074
+ * Value type is <<path,path>>
1075
+ * There is no default value for this setting.
1076
+
1077
+ The keystore used to present a certificate to the server.
1078
+ It can be either `.jks` or `.p12`
1079
+
1080
+ NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
1081
+
1082
+ [id="plugins-{type}s-{plugin}-ssl_keystore_type"]
1083
+ ===== `ssl_keystore_type`
1084
+
1085
+ * Value can be any of: `jks`, `pkcs12`
1086
+ * If not provided, the value will be inferred from the keystore filename.
1087
+
1088
+ The format of the keystore file. It must be either `jks` or `pkcs12`.
1089
+
1090
+ [id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
1091
+ ===== `ssl_supported_protocols`
1092
+
1093
+ * Value type is <<string,string>>
1094
+ * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
1095
+ * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
1096
+ `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
1097
+
1098
+ List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
1099
+
1100
+ For Java 8 `'TLSv1.3'` is supported only since **8u262** (AdoptOpenJDK), but requires that you set the
1101
+ `LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
1102
+
1103
+ NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
1104
+ the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
1105
+ the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
1106
+
1107
+ [id="plugins-{type}s-{plugin}-ssl_truststore_password"]
1108
+ ===== `ssl_truststore_password`
1109
+
1110
+ * Value type is <<password,password>>
1111
+ * There is no default value for this setting.
1112
+
1113
+ Set the truststore password
1114
+
1115
+ [id="plugins-{type}s-{plugin}-ssl_truststore_path"]
1116
+ ===== `ssl_truststore_path`
1117
+
1118
+ * Value type is <<path,path>>
1119
+ * There is no default value for this setting.
1120
+
1121
+ The truststore to validate the server's certificate.
1122
+ It can be either `.jks` or `.p12`.
1123
+
1124
+ NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> at the same time.
1125
+
1126
+ [id="plugins-{type}s-{plugin}-ssl_truststore_type"]
1127
+ ===== `ssl_truststore_type`
1128
+
1129
+ * Value can be any of: `jks`, `pkcs12`
1130
+ * If not provided, the value will be inferred from the truststore filename.
1131
+
1132
+ The format of the truststore file. It must be either `jks` or `pkcs12`.
1133
+
1134
+ [id="plugins-{type}s-{plugin}-ssl_verification_mode"]
1135
+ ===== `ssl_verification_mode`
1136
+
1137
+ * Value can be any of: `full`, `none`
1138
+ * Default value is `full`
1139
+
1140
+ Defines how to verify the certificates presented by another party in the TLS connection:
1141
+
1142
+ `full` validates that the server certificate has an issue date that’s within
1143
+ the not_before and not_after dates; chains to a trusted Certificate Authority (CA), and
1144
+ has a hostname or IP address that matches the names within the certificate.
1145
+
1146
+ `none` performs no certificate validation.
1147
+
1148
+ WARNING: Setting certificate verification to `none` disables many security benefits of SSL/TLS, which is very dangerous. For more information on disabling certificate verification please read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
1149
+
1150
+ [id="plugins-{type}s-{plugin}-template"]
1151
+ ===== `template`
1152
+
1153
+ * Value type is <<path,path>>
1154
+ * There is no default value for this setting.
1155
+
1156
+ You can set the path to your own template here, if you so desire.
1157
+ If not set, the included template will be used.
1158
+
1159
+ [id="plugins-{type}s-{plugin}-template_api"]
1160
+ ===== `template_api`
1161
+
1162
+ * Value can be any of: `auto`, `legacy`, `composable`
1163
+ * Default value is `auto`
1164
+
1165
+ The default setting of `auto` will use
1166
+ {ref}/index-templates.html[index template API] to create index template, if the
1167
+ Elasticsearch cluster is running Elasticsearch version `8.0.0` or higher,
1168
+ and use {ref}/indices-templates-v1.html[legacy template API] otherwise.
1169
+
1170
+ Setting this flag to `legacy` will use legacy template API to create index template.
1171
+ Setting this flag to `composable` will use index template API to create index template.
1172
+
1173
+ NOTE: The format of template provided to <<plugins-{type}s-{plugin}-template>> needs to match the template API being used.
1174
+
1175
+ [id="plugins-{type}s-{plugin}-template_name"]
1176
+ ===== `template_name`
1177
+
1178
+ * Value type is <<string,string>>
1179
+ * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
1180
+ ** ECS Compatibility disabled: `logstash`
1181
+ ** ECS Compatibility enabled: `ecs-logstash`
1182
+
1183
+
1184
+ This configuration option defines how the template is named inside Elasticsearch.
1185
+ Note that if you have used the template management features and subsequently
1186
+ change this, you will need to prune the old template manually, e.g.
1187
+
1188
+ `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
1189
+
1190
+ where `OldTemplateName` is whatever the former setting was.
1191
+
1192
+ [id="plugins-{type}s-{plugin}-template_overwrite"]
1193
+ ===== `template_overwrite`
1194
+
1195
+ * Value type is <<boolean,boolean>>
1196
+ * Default value is `false`
1197
+
1198
+ The template_overwrite option will always overwrite the indicated template
1199
+ in Elasticsearch with either the one indicated by template or the included one.
1200
+ This option is set to false by default. If you always want to stay up to date
1201
+ with the template provided by Logstash, this option could be very useful to you.
1202
+ Likewise, if you have your own template file managed by puppet, for example, and
1203
+ you wanted to be able to update it regularly, this option could help there as well.
1204
+
1205
+ Please note that if you are using your own customized version of the Logstash
1206
+ template (logstash), setting this to true will make Logstash to overwrite
1207
+ the "logstash" template (i.e. removing all customized settings)
1208
+
1209
+ [id="plugins-{type}s-{plugin}-timeout"]
1210
+ ===== `timeout`
1211
+
1212
+ * Value type is <<number,number>>
1213
+ * Default value is `60`
1214
+
1215
+ Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
1216
+ a timeout occurs, the request will be retried.
1217
+
1218
+ [id="plugins-{type}s-{plugin}-upsert"]
1219
+ ===== `upsert`
1220
+
1221
+ * Value type is <<string,string>>
1222
+ * Default value is `""`
1223
+
1224
+ Set upsert content for update mode.
1225
+ Create a new document with this parameter as json string if `document_id` doesn't exists
1226
+
1227
+ [id="plugins-{type}s-{plugin}-user"]
1228
+ ===== `user`
1229
+
1230
+ * Value type is <<string,string>>
1231
+ * There is no default value for this setting.
1232
+
1233
+ Username to authenticate to a secure Elasticsearch cluster
1234
+
1235
+ [id="plugins-{type}s-{plugin}-validate_after_inactivity"]
1236
+ ===== `validate_after_inactivity`
1237
+
1238
+ * Value type is <<number,number>>
1239
+ * Default value is `10000`
1240
+
1241
+ How long to wait before checking for a stale connection to determine if a keepalive request is needed.
1242
+ Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly.
1243
+
1244
+ This client is based on Apache Commons. Here's how the
1245
+ https://hc.apache.org/httpcomponents-client-4.5.x/current/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache
1246
+ Commons documentation] describes this option: "Defines period of inactivity in
1247
+ milliseconds after which persistent connections must be re-validated prior to
1248
+ being leased to the consumer. Non-positive value passed to this method disables
1249
+ connection validation. This check helps detect connections that have become
1250
+ stale (half-closed) while kept inactive in the pool."
1251
+
1252
+ [id="plugins-{type}s-{plugin}-version"]
1253
+ ===== `version`
1254
+
1255
+ * Value type is <<string,string>>
1256
+ * There is no default value for this setting.
1257
+
1258
+ The version to use for indexing. Use sprintf syntax like `%{my_version}` to use
1259
+ a field value here. See the
1260
+ https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support
1261
+ blog] for more information.
1262
+
1263
+ [id="plugins-{type}s-{plugin}-version_type"]
1264
+ ===== `version_type`
1265
+
1266
+ * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force`
1267
+ * There is no default value for this setting.
1268
+
1269
+ The version_type to use for indexing. See the
1270
+ https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support
1271
+ blog] and {ref}/docs-index_.html#_version_types[Version types] in the
1272
+ Elasticsearch documentation.
1273
+
1274
+ [id="plugins-{type}s-{plugin}-deprecated-options"]
1275
+ ==== Elasticsearch Output Deprecated Configuration Options
1276
+
1277
+ This plugin supports the following deprecated configurations.
1278
+
1279
+ WARNING: Deprecated options are subject to removal in future releases.
1280
+
1281
+ [cols="<,<,<",options="header",]
1282
+ |=======================================================================
1283
+ |Setting|Input type|Replaced by
1284
+ | <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
1285
+ | <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_keystore_path>>
1286
+ | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_keystore_password>>
1287
+ | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_enabled>>
1288
+ | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_verification_mode>>
1289
+ | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_truststore_path>>
1290
+ | <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_truststore_password>>
1291
+ |=======================================================================
1292
+
1293
+
1294
+ [id="plugins-{type}s-{plugin}-cacert"]
1295
+ ===== `cacert`
1296
+ deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
1297
+
1298
+ * Value type is a list of <<path,path>>
1299
+ * There is no default value for this setting.
1300
+
1301
+ The .cer or .pem file to validate the server's certificate.
1302
+
1303
+ [id="plugins-{type}s-{plugin}-keystore"]
1304
+ ===== `keystore`
1305
+ deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_path>>]
1306
+
1307
+ * Value type is <<path,path>>
1308
+ * There is no default value for this setting.
1309
+
1310
+ The keystore used to present a certificate to the server.
1311
+ It can be either .jks or .p12
1312
+
1313
+ NOTE: You cannot use this setting and <<plugins-{type}s-{plugin}-ssl_certificate>> at the same time.
1314
+
1315
+ [id="plugins-{type}s-{plugin}-keystore_password"]
1316
+ ===== `keystore_password`
1317
+ deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_password>>]
1318
+
1319
+ * Value type is <<password,password>>
1320
+ * There is no default value for this setting.
1321
+
1322
+ Set the keystore password
1323
+
1324
+ [id="plugins-{type}s-{plugin}-ssl"]
1325
+ ===== `ssl`
1326
+ deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
1327
+
1328
+ * Value type is <<boolean,boolean>>
1329
+ * There is no default value for this setting.
1330
+
1331
+ Enable SSL/TLS secured communication to Elasticsearch cluster.
1332
+ Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
1333
+ If no explicit protocol is specified plain HTTP will be used.
1334
+
1335
+ [id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
1336
+ ===== `ssl_certificate_verification`
1337
+ deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_verification_mode>>]
1338
+
1339
+ * Value type is <<boolean,boolean>>
1340
+ * Default value is `true`
1341
+
1342
+ Option to validate the server's certificate. Disabling this severely compromises security.
1343
+ For more information on disabling certificate verification please read
1344
+ https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
1345
+
1346
+ [id="plugins-{type}s-{plugin}-truststore"]
1347
+ ===== `truststore`
1348
+ deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_truststore_path>>]
1349
+
1350
+ * Value type is <<path,path>>
1351
+ * There is no default value for this setting.
1352
+
1353
+ The truststore to validate the server's certificate.
1354
+ It can be either `.jks` or `.p12`.
1355
+ Use either `:truststore` or `:cacert`.
1356
+
1357
+ [id="plugins-{type}s-{plugin}-truststore_password"]
1358
+ ===== `truststore_password`
1359
+ deprecated[11.14.0, Replaced by <<plugins-{type}s-{plugin}-ssl_truststore_password>>]
1360
+
1361
+ * Value type is <<password,password>>
1362
+ * There is no default value for this setting.
1363
+
1364
+ Set the truststore password
1365
+
1366
+ [id="plugins-{type}s-{plugin}-common-options"]
1367
+ include::{include_path}/{type}.asciidoc[]
1368
+
1369
+ :no_codec!: