fluent-plugin-elasticsearch 4.0.5 → 4.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/.travis.yml +1 -1
- data/Gemfile +2 -1
- data/History.md +27 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +99 -28
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/gemfiles/Gemfile.elasticsearch.v6 +1 -1
- data/gemfiles/{Gemfile.ilm → Gemfile.without.ilm} +1 -2
- data/lib/fluent/plugin/elasticsearch_index_template.rb +1 -1
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +76 -36
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_error_handler.rb +1 -1
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +1 -1
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +172 -1
- data/test/plugin/test_in_elasticsearch.rb +1 -1
- data/test/plugin/test_out_elasticsearch.rb +329 -6
- data/test/plugin/test_out_elasticsearch_dynamic.rb +23 -5
- metadata +8 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a12dc6596a56b7486d45129b64187021215433d2e7aa5428e96712fb63a345b5
|
|
4
|
+
data.tar.gz: 6fe5296f4a0afa23e863399f6d1cc9eeaa8f70c55e5cb979ca8db1bbf7a8988f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4f5d200eb1dee56f9f4fc05efc46cdbf440e0e2df7152e527985b124ac91ff74b0d263bee36f69fddec9f5ccc3730032145d51a95cc22fc00817f909fd4c2a7a
|
|
7
|
+
data.tar.gz: 11b0453ea6cbcfaf46e71866a96daf4ceba8f3877dd23e7aa124ea214f056b1b7f56bd8082ae273883a7403a2f1752c64beb377a2a3564c1d30757a8a5f101fa
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
name: Autocloser
|
|
2
|
+
on: [issues]
|
|
3
|
+
jobs:
|
|
4
|
+
autoclose:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
steps:
|
|
7
|
+
- name: Autoclose issues that did not follow issue template
|
|
8
|
+
uses: roots/issue-closer-action@v1.1
|
|
9
|
+
with:
|
|
10
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
11
|
+
issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
|
|
12
|
+
issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Testing on Ubuntu
|
|
2
|
+
on:
|
|
3
|
+
- push
|
|
4
|
+
- pull_request
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
|
12
|
+
os:
|
|
13
|
+
- ubuntu-latest
|
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v2
|
|
17
|
+
- uses: ruby/setup-ruby@v1
|
|
18
|
+
with:
|
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
|
20
|
+
- name: unit testing
|
|
21
|
+
env:
|
|
22
|
+
CI: true
|
|
23
|
+
run: |
|
|
24
|
+
gem install bundler rake
|
|
25
|
+
bundle install --jobs 4 --retry 3
|
|
26
|
+
bundle exec rake test
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Testing on macOS
|
|
2
|
+
on:
|
|
3
|
+
- push
|
|
4
|
+
- pull_request
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
|
12
|
+
os:
|
|
13
|
+
- macOS-latest
|
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v2
|
|
17
|
+
- uses: ruby/setup-ruby@v1
|
|
18
|
+
with:
|
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
|
20
|
+
- name: unit testing
|
|
21
|
+
env:
|
|
22
|
+
CI: true
|
|
23
|
+
run: |
|
|
24
|
+
gem install bundler rake
|
|
25
|
+
bundle install --jobs 4 --retry 3
|
|
26
|
+
bundle exec rake test
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Testing on Windows
|
|
2
|
+
on:
|
|
3
|
+
- push
|
|
4
|
+
- pull_request
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
|
12
|
+
os:
|
|
13
|
+
- windows-latest
|
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v2
|
|
17
|
+
- uses: ruby/setup-ruby@v1
|
|
18
|
+
with:
|
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
|
20
|
+
- name: unit testing
|
|
21
|
+
env:
|
|
22
|
+
CI: true
|
|
23
|
+
run: |
|
|
24
|
+
gem install bundler rake
|
|
25
|
+
bundle install --jobs 4 --retry 3
|
|
26
|
+
bundle exec rake test
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
|
@@ -4,7 +4,8 @@ source 'https://rubygems.org'
|
|
|
4
4
|
gemspec
|
|
5
5
|
|
|
6
6
|
gem 'simplecov', require: false
|
|
7
|
-
gem 'coveralls', require: false
|
|
7
|
+
gem 'coveralls', ">= 0.8.0", require: false
|
|
8
8
|
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
|
9
9
|
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
|
10
|
+
gem "elasticsearch-xpack"
|
|
10
11
|
gem "oj"
|
data/History.md
CHANGED
|
@@ -2,6 +2,33 @@
|
|
|
2
2
|
|
|
3
3
|
### [Unreleased]
|
|
4
4
|
|
|
5
|
+
### 4.0.10
|
|
6
|
+
- filter_elasticsearch_genid: Use entire record as hash seed (#777)
|
|
7
|
+
- Suppress type in meta with suppress_type_name parameter (#774)
|
|
8
|
+
- filter\_elasticsearch\_genid: Add hash generation mechanism from events (#773)
|
|
9
|
+
- Clean up error text (#772)
|
|
10
|
+
- Use GitHub Actions badges instead of Travis' (#760)
|
|
11
|
+
- Add issue auto closer workflow (#759)
|
|
12
|
+
- Document required permissions (#757)
|
|
13
|
+
|
|
14
|
+
### 4.0.9
|
|
15
|
+
- Add possibility to configure multiple ILM policies (#753)
|
|
16
|
+
- Document required permissions (#757)
|
|
17
|
+
|
|
18
|
+
### 4.0.8
|
|
19
|
+
- Handle compressable connection usable state (#743)
|
|
20
|
+
- Use newer tls protocol versions (#739)
|
|
21
|
+
- Add GitHub Actions file (#740)
|
|
22
|
+
|
|
23
|
+
### 4.0.7
|
|
24
|
+
- Added http_backend_excon_nonblock config in out_elasticsearch (#733)
|
|
25
|
+
|
|
26
|
+
### 4.0.6
|
|
27
|
+
- Add fallback mechanism for handling to detect es version (#730)
|
|
28
|
+
- Remove needless section (#728)
|
|
29
|
+
- Handle exception if index already exists (#727)
|
|
30
|
+
- Tweak test cases (#726)
|
|
31
|
+
|
|
5
32
|
### 4.0.5
|
|
6
33
|
- add logstash_dateformat as placeholder (#718)
|
|
7
34
|
- Tweak travis.yml for suppressing validator warnings and add CI for Linux Arm64 architecture and macOS 10.14 (#724)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
## Index
|
|
2
|
+
|
|
3
|
+
* [Usage](#usage)
|
|
4
|
+
* [Configuration](#configuration)
|
|
5
|
+
+ [hash_id_key](#hash_id_key)
|
|
6
|
+
+ [include_tag_in_seed](#include_tag_in_seed)
|
|
7
|
+
+ [include_time_in_seed](#include_time_in_seed)
|
|
8
|
+
+ [use_record_as_seed](#use_record_as_seed)
|
|
9
|
+
+ [use_entire_record](#use_entire_record)
|
|
10
|
+
+ [record_keys](#record_keys)
|
|
11
|
+
+ [separator](#separator)
|
|
12
|
+
+ [hash_type](#hash_type)
|
|
13
|
+
* [Advanced Usage](#advanced-usage)
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
<source>
|
|
21
|
+
@type elasticsearch_genid
|
|
22
|
+
hash_id_key _hash
|
|
23
|
+
include_tag_in_seed false
|
|
24
|
+
include_time_in_seed false
|
|
25
|
+
use_record_as_seed false
|
|
26
|
+
use_entire_record false
|
|
27
|
+
record_keys []
|
|
28
|
+
separator _
|
|
29
|
+
hash_type md5
|
|
30
|
+
</match>
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Configuration
|
|
34
|
+
|
|
35
|
+
### hash_id_key
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
hash_id_key _id
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
You can specify generated hash storing key.
|
|
42
|
+
|
|
43
|
+
### include_tag_in_seed
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
include_tag_in_seed true
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
You can specify to use tag for hash generation seed.
|
|
50
|
+
|
|
51
|
+
### include_time_in_seed
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
include_time_in_seed true
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
You can specify to use time for hash generation seed.
|
|
58
|
+
|
|
59
|
+
### use_record_as_seed
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
use_record_as_seed true
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
You can specify to use record in events for hash generation seed. This parameter should be used with [record_keys](#record_keys) parameter in practice.
|
|
66
|
+
|
|
67
|
+
### record_keys
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
record_keys request_id,pipeline_id
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
You can specify keys which are record in events for hash generation seed. This parameter should be used with [use_record_as_seed](#use_record_as_seed) parameter in practice.
|
|
74
|
+
|
|
75
|
+
### use_entire_record
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
use_entire_record true
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
You can specify to use entire record in events for hash generation seed.
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
### separator
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
separator |
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
You can specify separator charactor to creating seed for hash generation.
|
|
91
|
+
|
|
92
|
+
### hash_type
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
hash_type sha1
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
You can specify hash algorithm.
|
|
99
|
+
|
|
100
|
+
## Advanced Usage
|
|
101
|
+
|
|
102
|
+
Elasticsearch GenID plugin can handle record contents differing with the following parameters:
|
|
103
|
+
|
|
104
|
+
```aconf
|
|
105
|
+
<filter the.awesome.your.routing.tag>
|
|
106
|
+
@type elasticsearch_genid
|
|
107
|
+
use_entire_record true
|
|
108
|
+
hash_type sha1
|
|
109
|
+
hash_id_key _hash
|
|
110
|
+
separator _
|
|
111
|
+
inc_time_as_key true
|
|
112
|
+
inc_tag_as_key true
|
|
113
|
+
</filter>
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The above configuration can handle tag, time, and record differing and generate different base64 encoded hash per record.
|
data/README.md
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
|
|
2
2
|
|
|
3
3
|
[](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
5
7
|
[](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
|
|
6
8
|
[](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
|
|
7
9
|
|
|
@@ -31,6 +33,7 @@ Current maintainers: @cosmo0920
|
|
|
31
33
|
+ [time_key_exclude_timestamp](#time_key_exclude_timestamp)
|
|
32
34
|
+ [include_timestamp](#include_timestamp)
|
|
33
35
|
+ [utc_index](#utc_index)
|
|
36
|
+
+ [suppress_type_name](#suppress_type_name)
|
|
34
37
|
+ [target_index_key](#target_index_key)
|
|
35
38
|
+ [target_type_key](#target_type_key)
|
|
36
39
|
+ [template_name](#template_name)
|
|
@@ -45,6 +48,7 @@ Current maintainers: @cosmo0920
|
|
|
45
48
|
+ [templates](#templates)
|
|
46
49
|
+ [max_retry_putting_template](#max_retry_putting_template)
|
|
47
50
|
+ [fail_on_putting_template_retry_exceed](#fail_on_putting_template_retry_exceed)
|
|
51
|
+
+ [fail_on_detecting_es_version_retry_exceed](#fail_on_detecting_es_version_retry_exceed)
|
|
48
52
|
+ [max_retry_get_es_version](#max_retry_get_es_version)
|
|
49
53
|
+ [request_timeout](#request_timeout)
|
|
50
54
|
+ [reload_connections](#reload_connections)
|
|
@@ -65,6 +69,7 @@ Current maintainers: @cosmo0920
|
|
|
65
69
|
+ [content_type](#content_type)
|
|
66
70
|
+ [include_index_in_url](#include_index_in_url)
|
|
67
71
|
+ [http_backend](#http_backend)
|
|
72
|
+
+ [http_backend_excon_nonblock](#http_backend_excon_nonblock)
|
|
68
73
|
+ [prefer_oj_serializer](#prefer_oj_serializer)
|
|
69
74
|
+ [compression_level](#compression_level)
|
|
70
75
|
+ [Client/host certificate options](#clienthost-certificate-options)
|
|
@@ -91,15 +96,17 @@ Current maintainers: @cosmo0920
|
|
|
91
96
|
+ [enable_ilm](#enable_ilm)
|
|
92
97
|
+ [ilm_policy_id](#ilm_policy_id)
|
|
93
98
|
+ [ilm_policy](#ilm_policy)
|
|
99
|
+
+ [ilm_policies](#ilm_policies)
|
|
94
100
|
+ [ilm_policy_overwrite](#ilm_policy_overwrite)
|
|
95
101
|
+ [truncate_caches_interval](#truncate_caches_interval)
|
|
96
102
|
* [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
|
|
103
|
+
* [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
|
|
104
|
+
* [Elasticsearch permissions](#elasticsearch-permissions)
|
|
97
105
|
* [Troubleshooting](#troubleshooting)
|
|
98
106
|
+ [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
|
|
99
107
|
+ [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
|
|
100
108
|
+ [Cannot connect TLS enabled reverse Proxy](#cannot-connect-tls-enabled-reverse-proxy)
|
|
101
109
|
+ [Declined logs are resubmitted forever, why?](#declined-logs-are-resubmitted-forever-why)
|
|
102
|
-
+ [Suggested to increase flush_thread_count, why?](#suggested-to-increase-flush_thread_count-why)
|
|
103
110
|
+ [Suggested to install typhoeus gem, why?](#suggested-to-install-typhoeus-gem-why)
|
|
104
111
|
+ [Stopped to send events on k8s, why?](#stopped-to-send-events-on-k8s-why)
|
|
105
112
|
+ [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
|
|
@@ -348,6 +355,20 @@ utc_index true
|
|
|
348
355
|
|
|
349
356
|
By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
|
|
350
357
|
|
|
358
|
+
### suppress_type_name
|
|
359
|
+
|
|
360
|
+
In Elasticsearch 7.x, Elasticsearch cluster complains the following types removal warnings:
|
|
361
|
+
|
|
362
|
+
```json
|
|
363
|
+
{"type": "deprecation", "timestamp": "2020-07-03T08:02:20,830Z", "level": "WARN", "component": "o.e.d.a.b.BulkRequestParser", "cluster.name": "docker-cluster", "node.name": "70dd5c6b94c3", "message": "[types removal] Specifying types in bulk requests is deprecated.", "cluster.uuid": "NoJJmtzfTtSzSMv0peG8Wg", "node.id": "VQ-PteHmTVam2Pnbg7xWHw" }
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
This can be suppressed with:
|
|
367
|
+
|
|
368
|
+
```
|
|
369
|
+
suppress_type_name true
|
|
370
|
+
```
|
|
371
|
+
|
|
351
372
|
### target_index_key
|
|
352
373
|
|
|
353
374
|
Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
|
|
@@ -502,6 +523,23 @@ If you have multiple output plugin, you could use this property to do not fail o
|
|
|
502
523
|
fail_on_putting_template_retry_exceed false # defaults to true
|
|
503
524
|
```
|
|
504
525
|
|
|
526
|
+
### fail_on_detecting_es_version_retry_exceed
|
|
527
|
+
|
|
528
|
+
Indicates whether to fail when `max_retry_get_es_version` is exceeded.
|
|
529
|
+
If you want to use fallback mechanism for obtaining ELasticsearch version, you could use this property to do not fail on fluentd statup.
|
|
530
|
+
|
|
531
|
+
```
|
|
532
|
+
fail_on_detecting_es_version_retry_exceed false
|
|
533
|
+
```
|
|
534
|
+
|
|
535
|
+
And the following parameters should be working with:
|
|
536
|
+
|
|
537
|
+
```
|
|
538
|
+
verify_es_version_at_startup true
|
|
539
|
+
max_retry_get_es_version 2 # greater than 0.
|
|
540
|
+
default_elasticsearch_version 7 # This version is used when occurring fallback.
|
|
541
|
+
```
|
|
542
|
+
|
|
505
543
|
### max_retry_get_es_version
|
|
506
544
|
|
|
507
545
|
You can specify times of retry obtaining Elasticsearch version.
|
|
@@ -799,6 +837,21 @@ Default value is `excon` which is default http_backend of elasticsearch plugin.
|
|
|
799
837
|
http_backend typhoeus
|
|
800
838
|
```
|
|
801
839
|
|
|
840
|
+
### http_backend_excon_nonblock
|
|
841
|
+
|
|
842
|
+
With `http_backend_excon_nonblock false`, elasticsearch plugin use excon with nonblock=false.
|
|
843
|
+
If you use elasticsearch plugin with jRuby for https, you may need to consider to set `false` to avoid follwoing problems.
|
|
844
|
+
- https://github.com/geemus/excon/issues/106
|
|
845
|
+
- https://github.com/jruby/jruby-ossl/issues/19
|
|
846
|
+
|
|
847
|
+
But for all other case, it strongly reccomend to set `true` to avoid process hangin problem reported in https://github.com/uken/fluent-plugin-elasticsearch/issues/732
|
|
848
|
+
|
|
849
|
+
Default value is `true`.
|
|
850
|
+
|
|
851
|
+
```
|
|
852
|
+
http_backend_excon_nonblock false
|
|
853
|
+
```
|
|
854
|
+
|
|
802
855
|
### compression_level
|
|
803
856
|
You can add gzip compression of output data. In this case `default_compression`, `best_compression` or `best speed` option should be chosen.
|
|
804
857
|
By default there is no compression, default value for this option is `no_compression`
|
|
@@ -848,7 +901,11 @@ ssl_min_version TLSv1_2
|
|
|
848
901
|
|
|
849
902
|
Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
|
|
850
903
|
|
|
851
|
-
If they are *not* specified in the Elasticsearch plugin configuration,
|
|
904
|
+
If they are *not* specified in the Elasticsearch plugin configuration, `ssl_max_version` and `ssl_min_version` is set up with:
|
|
905
|
+
|
|
906
|
+
In Elasticsearch plugin v4.0.8 or later with Ruby 2.5 or later environment, `ssl_max_version` should be `TLSv1_3` and `ssl_min_version` should be `TLSv1_2`.
|
|
907
|
+
|
|
908
|
+
From Elasticsearch plugin v4.0.4 to v4.0.7 with Ruby 2.5 or later environment, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
|
|
852
909
|
|
|
853
910
|
|
|
854
911
|
### Proxy Support
|
|
@@ -946,7 +1003,7 @@ If you use Fluentd directly, you must pass the following lines as Fluentd comman
|
|
|
946
1003
|
|
|
947
1004
|
```
|
|
948
1005
|
sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
|
|
949
|
-
$ fluentd -r $sniffer
|
|
1006
|
+
$ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
|
|
950
1007
|
```
|
|
951
1008
|
|
|
952
1009
|
### Reload After
|
|
@@ -1176,6 +1233,14 @@ Default value is `{}`.
|
|
|
1176
1233
|
|
|
1177
1234
|
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
|
1178
1235
|
|
|
1236
|
+
## ilm_policies
|
|
1237
|
+
|
|
1238
|
+
A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
|
|
1239
|
+
|
|
1240
|
+
Default value is `{}`.
|
|
1241
|
+
|
|
1242
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
|
1243
|
+
|
|
1179
1244
|
## ilm_policy_overwrite
|
|
1180
1245
|
|
|
1181
1246
|
Specify whether overwriting ilm policy or not.
|
|
@@ -1196,6 +1261,36 @@ Default value is `nil`.
|
|
|
1196
1261
|
|
|
1197
1262
|
See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
|
|
1198
1263
|
|
|
1264
|
+
## Configuration - Elasticsearch Filter GenID
|
|
1265
|
+
|
|
1266
|
+
See [Elasticsearch Filter GenID document](README.ElasticsearchGenID.md)
|
|
1267
|
+
|
|
1268
|
+
## Elasticsearch permissions
|
|
1269
|
+
|
|
1270
|
+
If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
|
|
1271
|
+
|
|
1272
|
+
The set of required permissions are the following:
|
|
1273
|
+
|
|
1274
|
+
```json
|
|
1275
|
+
"cluster": ["manage_index_templates", "monitor", "manage_ilm"],
|
|
1276
|
+
"indices": [
|
|
1277
|
+
{
|
|
1278
|
+
"names": [ "*" ],
|
|
1279
|
+
"privileges": ["write","create","delete","create_index","manage","manage_ilm"]
|
|
1280
|
+
}
|
|
1281
|
+
]
|
|
1282
|
+
```
|
|
1283
|
+
|
|
1284
|
+
These permissions can be narrowed down by:
|
|
1285
|
+
|
|
1286
|
+
- Setting a more specific pattern for indices under the `names` field
|
|
1287
|
+
- Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
|
|
1288
|
+
- Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
|
|
1289
|
+
features in the plugin configuration
|
|
1290
|
+
|
|
1291
|
+
The list of privileges along with their description can be found in
|
|
1292
|
+
[security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
|
|
1293
|
+
|
|
1199
1294
|
## Troubleshooting
|
|
1200
1295
|
|
|
1201
1296
|
### Cannot send events to Elasticsearch
|
|
@@ -1445,30 +1540,6 @@ The following configuration uses label:
|
|
|
1445
1540
|
</label>
|
|
1446
1541
|
```
|
|
1447
1542
|
|
|
1448
|
-
### Suggested to increase flush_thread_count, why?
|
|
1449
|
-
|
|
1450
|
-
fluent-plugin-elasticsearch default behavior has a possibility to cause events traffic jam.
|
|
1451
|
-
When users use `flush_thread_count` = 1, ES plugin retries to send events if connection errors are disappeared.
|
|
1452
|
-
|
|
1453
|
-
To prevent the following warning and sending events blocking, you must specify `flush_thread_count` >= 2:
|
|
1454
|
-
|
|
1455
|
-
```log
|
|
1456
|
-
2018-12-24 14:32:06 +0900 [warn]: #0 To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'.
|
|
1457
|
-
```
|
|
1458
|
-
|
|
1459
|
-
```aconf
|
|
1460
|
-
<match out.elasticsearch.**>
|
|
1461
|
-
@type elasticsearch
|
|
1462
|
-
host localhost
|
|
1463
|
-
port 9200
|
|
1464
|
-
# ...
|
|
1465
|
-
<buffer tag>
|
|
1466
|
-
@type memory # or file
|
|
1467
|
-
flush_thread_count 4
|
|
1468
|
-
</buffer>
|
|
1469
|
-
</match>
|
|
1470
|
-
```
|
|
1471
|
-
|
|
1472
1543
|
### Suggested to install typhoeus gem, why?
|
|
1473
1544
|
|
|
1474
1545
|
fluent-plugin-elasticsearch doesn't depend on typhoeus gem by default.
|