fluent-plugin-elasticsearch 4.0.6 → 4.0.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/History.md +25 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +81 -3
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +67 -32
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +423 -5
- data/test/plugin/test_out_elasticsearch_dynamic.rb +21 -3
- metadata +7 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c90e0b347a5c90ec23231df927384dadcc4b91b813a4e2c11df23539dfe05628
|
|
4
|
+
data.tar.gz: d6e296c8006870a5f85bd0b97294c69d797fc3f40b574ede7645fcd808ae6356
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8466c4e3a13ef5c5ca1bda536a0baf7044fdaa0bc3ae1e1c7e6d0e6cf3dc7596d2b68e3f61ed2fb788f8ef2d8392939b8f63fd6aedc53e5818725bc6e0910aa6
|
|
7
|
+
data.tar.gz: b0572e50952ce49e44b280e94764d0d204d482e8e89cc2124e9a3025c1bd4cf7debe5625ad141337e5f81e6b6816286ee81f931f6b782f0a8cffcbfa0ed12dc3
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
name: Autocloser
|
|
2
|
+
on: [issues]
|
|
3
|
+
jobs:
|
|
4
|
+
autoclose:
|
|
5
|
+
runs-on: ubuntu-latest
|
|
6
|
+
steps:
|
|
7
|
+
- name: Autoclose issues that did not follow issue template
|
|
8
|
+
uses: roots/issue-closer-action@v1.1
|
|
9
|
+
with:
|
|
10
|
+
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
11
|
+
issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
|
|
12
|
+
issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Testing on Ubuntu
|
|
2
|
+
on:
|
|
3
|
+
- push
|
|
4
|
+
- pull_request
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
|
12
|
+
os:
|
|
13
|
+
- ubuntu-latest
|
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v2
|
|
17
|
+
- uses: ruby/setup-ruby@v1
|
|
18
|
+
with:
|
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
|
20
|
+
- name: unit testing
|
|
21
|
+
env:
|
|
22
|
+
CI: true
|
|
23
|
+
run: |
|
|
24
|
+
gem install bundler rake
|
|
25
|
+
bundle install --jobs 4 --retry 3
|
|
26
|
+
bundle exec rake test
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Testing on macOS
|
|
2
|
+
on:
|
|
3
|
+
- push
|
|
4
|
+
- pull_request
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
|
12
|
+
os:
|
|
13
|
+
- macOS-latest
|
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v2
|
|
17
|
+
- uses: ruby/setup-ruby@v1
|
|
18
|
+
with:
|
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
|
20
|
+
- name: unit testing
|
|
21
|
+
env:
|
|
22
|
+
CI: true
|
|
23
|
+
run: |
|
|
24
|
+
gem install bundler rake
|
|
25
|
+
bundle install --jobs 4 --retry 3
|
|
26
|
+
bundle exec rake test
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Testing on Windows
|
|
2
|
+
on:
|
|
3
|
+
- push
|
|
4
|
+
- pull_request
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
|
12
|
+
os:
|
|
13
|
+
- windows-latest
|
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v2
|
|
17
|
+
- uses: ruby/setup-ruby@v1
|
|
18
|
+
with:
|
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
|
20
|
+
- name: unit testing
|
|
21
|
+
env:
|
|
22
|
+
CI: true
|
|
23
|
+
run: |
|
|
24
|
+
gem install bundler rake
|
|
25
|
+
bundle install --jobs 4 --retry 3
|
|
26
|
+
bundle exec rake test
|
data/History.md
CHANGED
|
@@ -2,6 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
### [Unreleased]
|
|
4
4
|
|
|
5
|
+
### 4.0.11
|
|
6
|
+
- Add custom and time placeholders combination testcase for ILM (#781)
|
|
7
|
+
- Really support ILM on `logstash_format` enabled environment (#779)
|
|
8
|
+
|
|
9
|
+
### 4.0.10
|
|
10
|
+
- filter_elasticsearch_genid: Use entire record as hash seed (#777)
|
|
11
|
+
- Suppress type in meta with suppress_type_name parameter (#774)
|
|
12
|
+
- filter\_elasticsearch\_genid: Add hash generation mechanism from events (#773)
|
|
13
|
+
- Clean up error text (#772)
|
|
14
|
+
- Use GitHub Actions badges instead of Travis' (#760)
|
|
15
|
+
- Add issue auto closer workflow (#759)
|
|
16
|
+
- Document required permissions (#757)
|
|
17
|
+
|
|
18
|
+
### 4.0.9
|
|
19
|
+
- Add possibility to configure multiple ILM policies (#753)
|
|
20
|
+
- Document required permissions (#757)
|
|
21
|
+
|
|
22
|
+
### 4.0.8
|
|
23
|
+
- Handle compressable connection usable state (#743)
|
|
24
|
+
- Use newer tls protocol versions (#739)
|
|
25
|
+
- Add GitHub Actions file (#740)
|
|
26
|
+
|
|
27
|
+
### 4.0.7
|
|
28
|
+
- Added http_backend_excon_nonblock config in out_elasticsearch (#733)
|
|
29
|
+
|
|
5
30
|
### 4.0.6
|
|
6
31
|
- Add fallback mechanism for handling to detect es version (#730)
|
|
7
32
|
- Remove needless section (#728)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
## Index
|
|
2
|
+
|
|
3
|
+
* [Usage](#usage)
|
|
4
|
+
* [Configuration](#configuration)
|
|
5
|
+
+ [hash_id_key](#hash_id_key)
|
|
6
|
+
+ [include_tag_in_seed](#include_tag_in_seed)
|
|
7
|
+
+ [include_time_in_seed](#include_time_in_seed)
|
|
8
|
+
+ [use_record_as_seed](#use_record_as_seed)
|
|
9
|
+
+ [use_entire_record](#use_entire_record)
|
|
10
|
+
+ [record_keys](#record_keys)
|
|
11
|
+
+ [separator](#separator)
|
|
12
|
+
+ [hash_type](#hash_type)
|
|
13
|
+
* [Advanced Usage](#advanced-usage)
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
<source>
|
|
21
|
+
@type elasticsearch_genid
|
|
22
|
+
hash_id_key _hash
|
|
23
|
+
include_tag_in_seed false
|
|
24
|
+
include_time_in_seed false
|
|
25
|
+
use_record_as_seed false
|
|
26
|
+
use_entire_record false
|
|
27
|
+
record_keys []
|
|
28
|
+
separator _
|
|
29
|
+
hash_type md5
|
|
30
|
+
</match>
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Configuration
|
|
34
|
+
|
|
35
|
+
### hash_id_key
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
hash_id_key _id
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
You can specify generated hash storing key.
|
|
42
|
+
|
|
43
|
+
### include_tag_in_seed
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
include_tag_in_seed true
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
You can specify to use tag for hash generation seed.
|
|
50
|
+
|
|
51
|
+
### include_time_in_seed
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
include_time_in_seed true
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
You can specify to use time for hash generation seed.
|
|
58
|
+
|
|
59
|
+
### use_record_as_seed
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
use_record_as_seed true
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
You can specify to use record in events for hash generation seed. This parameter should be used with [record_keys](#record_keys) parameter in practice.
|
|
66
|
+
|
|
67
|
+
### record_keys
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
record_keys request_id,pipeline_id
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
You can specify keys which are record in events for hash generation seed. This parameter should be used with [use_record_as_seed](#use_record_as_seed) parameter in practice.
|
|
74
|
+
|
|
75
|
+
### use_entire_record
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
use_entire_record true
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
You can specify to use entire record in events for hash generation seed.
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
### separator
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
separator |
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
You can specify separator charactor to creating seed for hash generation.
|
|
91
|
+
|
|
92
|
+
### hash_type
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
hash_type sha1
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
You can specify hash algorithm.
|
|
99
|
+
|
|
100
|
+
## Advanced Usage
|
|
101
|
+
|
|
102
|
+
Elasticsearch GenID plugin can handle record contents differing with the following parameters:
|
|
103
|
+
|
|
104
|
+
```aconf
|
|
105
|
+
<filter the.awesome.your.routing.tag>
|
|
106
|
+
@type elasticsearch_genid
|
|
107
|
+
use_entire_record true
|
|
108
|
+
hash_type sha1
|
|
109
|
+
hash_id_key _hash
|
|
110
|
+
separator _
|
|
111
|
+
inc_time_as_key true
|
|
112
|
+
inc_tag_as_key true
|
|
113
|
+
</filter>
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The above configuration can handle tag, time, and record differing and generate different base64 encoded hash per record.
|
data/README.md
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
|
|
2
2
|
|
|
3
3
|
[](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
5
7
|
[](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
|
|
6
8
|
[](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
|
|
7
9
|
|
|
@@ -31,6 +33,7 @@ Current maintainers: @cosmo0920
|
|
|
31
33
|
+ [time_key_exclude_timestamp](#time_key_exclude_timestamp)
|
|
32
34
|
+ [include_timestamp](#include_timestamp)
|
|
33
35
|
+ [utc_index](#utc_index)
|
|
36
|
+
+ [suppress_type_name](#suppress_type_name)
|
|
34
37
|
+ [target_index_key](#target_index_key)
|
|
35
38
|
+ [target_type_key](#target_type_key)
|
|
36
39
|
+ [template_name](#template_name)
|
|
@@ -66,6 +69,7 @@ Current maintainers: @cosmo0920
|
|
|
66
69
|
+ [content_type](#content_type)
|
|
67
70
|
+ [include_index_in_url](#include_index_in_url)
|
|
68
71
|
+ [http_backend](#http_backend)
|
|
72
|
+
+ [http_backend_excon_nonblock](#http_backend_excon_nonblock)
|
|
69
73
|
+ [prefer_oj_serializer](#prefer_oj_serializer)
|
|
70
74
|
+ [compression_level](#compression_level)
|
|
71
75
|
+ [Client/host certificate options](#clienthost-certificate-options)
|
|
@@ -92,9 +96,12 @@ Current maintainers: @cosmo0920
|
|
|
92
96
|
+ [enable_ilm](#enable_ilm)
|
|
93
97
|
+ [ilm_policy_id](#ilm_policy_id)
|
|
94
98
|
+ [ilm_policy](#ilm_policy)
|
|
99
|
+
+ [ilm_policies](#ilm_policies)
|
|
95
100
|
+ [ilm_policy_overwrite](#ilm_policy_overwrite)
|
|
96
101
|
+ [truncate_caches_interval](#truncate_caches_interval)
|
|
97
102
|
* [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
|
|
103
|
+
* [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
|
|
104
|
+
* [Elasticsearch permissions](#elasticsearch-permissions)
|
|
98
105
|
* [Troubleshooting](#troubleshooting)
|
|
99
106
|
+ [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
|
|
100
107
|
+ [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
|
|
@@ -348,6 +355,20 @@ utc_index true
|
|
|
348
355
|
|
|
349
356
|
By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
|
|
350
357
|
|
|
358
|
+
### suppress_type_name
|
|
359
|
+
|
|
360
|
+
In Elasticsearch 7.x, Elasticsearch cluster complains the following types removal warnings:
|
|
361
|
+
|
|
362
|
+
```json
|
|
363
|
+
{"type": "deprecation", "timestamp": "2020-07-03T08:02:20,830Z", "level": "WARN", "component": "o.e.d.a.b.BulkRequestParser", "cluster.name": "docker-cluster", "node.name": "70dd5c6b94c3", "message": "[types removal] Specifying types in bulk requests is deprecated.", "cluster.uuid": "NoJJmtzfTtSzSMv0peG8Wg", "node.id": "VQ-PteHmTVam2Pnbg7xWHw" }
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
This can be suppressed with:
|
|
367
|
+
|
|
368
|
+
```
|
|
369
|
+
suppress_type_name true
|
|
370
|
+
```
|
|
371
|
+
|
|
351
372
|
### target_index_key
|
|
352
373
|
|
|
353
374
|
Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
|
|
@@ -816,6 +837,21 @@ Default value is `excon` which is default http_backend of elasticsearch plugin.
|
|
|
816
837
|
http_backend typhoeus
|
|
817
838
|
```
|
|
818
839
|
|
|
840
|
+
### http_backend_excon_nonblock
|
|
841
|
+
|
|
842
|
+
With `http_backend_excon_nonblock false`, elasticsearch plugin use excon with nonblock=false.
|
|
843
|
+
If you use elasticsearch plugin with jRuby for https, you may need to consider to set `false` to avoid follwoing problems.
|
|
844
|
+
- https://github.com/geemus/excon/issues/106
|
|
845
|
+
- https://github.com/jruby/jruby-ossl/issues/19
|
|
846
|
+
|
|
847
|
+
But for all other case, it strongly reccomend to set `true` to avoid process hangin problem reported in https://github.com/uken/fluent-plugin-elasticsearch/issues/732
|
|
848
|
+
|
|
849
|
+
Default value is `true`.
|
|
850
|
+
|
|
851
|
+
```
|
|
852
|
+
http_backend_excon_nonblock false
|
|
853
|
+
```
|
|
854
|
+
|
|
819
855
|
### compression_level
|
|
820
856
|
You can add gzip compression of output data. In this case `default_compression`, `best_compression` or `best speed` option should be chosen.
|
|
821
857
|
By default there is no compression, default value for this option is `no_compression`
|
|
@@ -865,7 +901,11 @@ ssl_min_version TLSv1_2
|
|
|
865
901
|
|
|
866
902
|
Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
|
|
867
903
|
|
|
868
|
-
If they are *not* specified in the Elasticsearch plugin configuration,
|
|
904
|
+
If they are *not* specified in the Elasticsearch plugin configuration, `ssl_max_version` and `ssl_min_version` is set up with:
|
|
905
|
+
|
|
906
|
+
In Elasticsearch plugin v4.0.8 or later with Ruby 2.5 or later environment, `ssl_max_version` should be `TLSv1_3` and `ssl_min_version` should be `TLSv1_2`.
|
|
907
|
+
|
|
908
|
+
From Elasticsearch plugin v4.0.4 to v4.0.7 with Ruby 2.5 or later environment, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
|
|
869
909
|
|
|
870
910
|
|
|
871
911
|
### Proxy Support
|
|
@@ -963,7 +1003,7 @@ If you use Fluentd directly, you must pass the following lines as Fluentd comman
|
|
|
963
1003
|
|
|
964
1004
|
```
|
|
965
1005
|
sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
|
|
966
|
-
$ fluentd -r $sniffer
|
|
1006
|
+
$ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
|
|
967
1007
|
```
|
|
968
1008
|
|
|
969
1009
|
### Reload After
|
|
@@ -1193,6 +1233,14 @@ Default value is `{}`.
|
|
|
1193
1233
|
|
|
1194
1234
|
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
|
1195
1235
|
|
|
1236
|
+
## ilm_policies
|
|
1237
|
+
|
|
1238
|
+
A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
|
|
1239
|
+
|
|
1240
|
+
Default value is `{}`.
|
|
1241
|
+
|
|
1242
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
|
1243
|
+
|
|
1196
1244
|
## ilm_policy_overwrite
|
|
1197
1245
|
|
|
1198
1246
|
Specify whether overwriting ilm policy or not.
|
|
@@ -1213,6 +1261,36 @@ Default value is `nil`.
|
|
|
1213
1261
|
|
|
1214
1262
|
See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
|
|
1215
1263
|
|
|
1264
|
+
## Configuration - Elasticsearch Filter GenID
|
|
1265
|
+
|
|
1266
|
+
See [Elasticsearch Filter GenID document](README.ElasticsearchGenID.md)
|
|
1267
|
+
|
|
1268
|
+
## Elasticsearch permissions
|
|
1269
|
+
|
|
1270
|
+
If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
|
|
1271
|
+
|
|
1272
|
+
The set of required permissions are the following:
|
|
1273
|
+
|
|
1274
|
+
```json
|
|
1275
|
+
"cluster": ["manage_index_templates", "monitor", "manage_ilm"],
|
|
1276
|
+
"indices": [
|
|
1277
|
+
{
|
|
1278
|
+
"names": [ "*" ],
|
|
1279
|
+
"privileges": ["write","create","delete","create_index","manage","manage_ilm"]
|
|
1280
|
+
}
|
|
1281
|
+
]
|
|
1282
|
+
```
|
|
1283
|
+
|
|
1284
|
+
These permissions can be narrowed down by:
|
|
1285
|
+
|
|
1286
|
+
- Setting a more specific pattern for indices under the `names` field
|
|
1287
|
+
- Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
|
|
1288
|
+
- Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
|
|
1289
|
+
features in the plugin configuration
|
|
1290
|
+
|
|
1291
|
+
The list of privileges along with their description can be found in
|
|
1292
|
+
[security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
|
|
1293
|
+
|
|
1216
1294
|
## Troubleshooting
|
|
1217
1295
|
|
|
1218
1296
|
### Cannot send events to Elasticsearch
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |s|
|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
|
6
|
-
s.version = '4.0.
|
|
6
|
+
s.version = '4.0.11'
|
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
|
@@ -10,7 +10,7 @@ module Fluent::Plugin
|
|
|
10
10
|
[:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
DEFAULT_VERSION = :
|
|
13
|
+
DEFAULT_VERSION = :TLSv1_2
|
|
14
14
|
METHODS_MAP = begin
|
|
15
15
|
# When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
|
|
16
16
|
map = {
|
|
@@ -48,8 +48,8 @@ module Fluent::Plugin
|
|
|
48
48
|
if USE_TLS_MINMAX_VERSION
|
|
49
49
|
case
|
|
50
50
|
when ssl_min_version.nil? && ssl_max_version.nil?
|
|
51
|
-
ssl_min_version = METHODS_MAP[
|
|
52
|
-
ssl_max_version = METHODS_MAP[
|
|
51
|
+
ssl_min_version = METHODS_MAP[:TLSv1_2]
|
|
52
|
+
ssl_max_version = METHODS_MAP[:TLSv1_3]
|
|
53
53
|
when ssl_min_version && ssl_max_version.nil?
|
|
54
54
|
raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
|
|
55
55
|
when ssl_min_version.nil? && ssl_max_version
|
|
@@ -7,6 +7,13 @@ module Fluent::Plugin
|
|
|
7
7
|
Fluent::Plugin.register_filter('elasticsearch_genid', self)
|
|
8
8
|
|
|
9
9
|
config_param :hash_id_key, :string, :default => '_hash'
|
|
10
|
+
config_param :include_tag_in_seed, :bool, :default => false
|
|
11
|
+
config_param :include_time_in_seed, :bool, :default => false
|
|
12
|
+
config_param :use_record_as_seed, :bool, :default => false
|
|
13
|
+
config_param :use_entire_record, :bool, :default => false
|
|
14
|
+
config_param :record_keys, :array, :default => []
|
|
15
|
+
config_param :separator, :string, :default => '_'
|
|
16
|
+
config_param :hash_type, :enum, list: [:md5, :sha1, :sha256, :sha512], :default => :sha1
|
|
10
17
|
|
|
11
18
|
def initialize
|
|
12
19
|
super
|
|
@@ -14,12 +21,57 @@ module Fluent::Plugin
|
|
|
14
21
|
|
|
15
22
|
def configure(conf)
|
|
16
23
|
super
|
|
24
|
+
|
|
25
|
+
if !@use_entire_record
|
|
26
|
+
if @record_keys.empty? && @use_record_as_seed
|
|
27
|
+
raise Fluent::ConfigError, "When using record as hash seed, users must specify `record_keys`."
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
if @use_record_as_seed
|
|
32
|
+
class << self
|
|
33
|
+
alias_method :filter, :filter_seed_as_record
|
|
34
|
+
end
|
|
35
|
+
else
|
|
36
|
+
class << self
|
|
37
|
+
alias_method :filter, :filter_simple
|
|
38
|
+
end
|
|
39
|
+
end
|
|
17
40
|
end
|
|
18
41
|
|
|
19
42
|
def filter(tag, time, record)
|
|
43
|
+
# for safety.
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def filter_simple(tag, time, record)
|
|
20
47
|
record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
|
|
21
48
|
record
|
|
22
49
|
end
|
|
23
50
|
|
|
51
|
+
def filter_seed_as_record(tag, time, record)
|
|
52
|
+
seed = ""
|
|
53
|
+
seed += tag + separator if @include_tag_in_seed
|
|
54
|
+
seed += time.to_s + separator if @include_time_in_seed
|
|
55
|
+
if @use_entire_record
|
|
56
|
+
record.each {|k,v| seed += "|#{k}|#{v}"}
|
|
57
|
+
else
|
|
58
|
+
seed += record_keys.map {|k| record[k]}.join(separator)
|
|
59
|
+
end
|
|
60
|
+
record[@hash_id_key] = Base64.strict_encode64(encode_hash(@hash_type, seed))
|
|
61
|
+
record
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def encode_hash(type, seed)
|
|
65
|
+
case type
|
|
66
|
+
when :md5
|
|
67
|
+
Digest::MD5.digest(seed)
|
|
68
|
+
when :sha1
|
|
69
|
+
Digest::SHA1.digest(seed)
|
|
70
|
+
when :sha256
|
|
71
|
+
Digest::SHA256.digest(seed)
|
|
72
|
+
when :sha512
|
|
73
|
+
Digest::SHA512.digest(seed)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
24
76
|
end
|
|
25
77
|
end
|