fluent-plugin-elasticsearch 4.0.7 → 4.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d6a25db7c9a8c6d5d31365939cb60092440a299f4bacb52f1c3896927aa501a7
4
- data.tar.gz: ed08dc5d2d526a9211a76dd36c0db7a90f1e50a17d6cc634790d3e6cba286175
3
+ metadata.gz: 12f88bef6f61ff3acbbd654c63a6402f7374678e78cf438f2026551246e9e0fc
4
+ data.tar.gz: 7c8ffbd9626bf8204820326ed7a3bb9157f0e3e6b406dbba51f2cb66c899b6ca
5
5
  SHA512:
6
- metadata.gz: c00a1d839a4adc7d47b3032099a81f6e73fe126ef0711d61518fe9034b915a39d9b6f50d19cc7a5dd39f8e903e243b745fc615a097794d4d456ba2a2a71a54c5
7
- data.tar.gz: 84501757defe9cd9e78f95843cc39f3cfa68a32c09ef0235764c2e2bc2bc9e37a59241bd40ba74a87de8bbf968ea172db5e6aed55b233833029683213b0040ec
6
+ metadata.gz: 1bc665321d8aed75d579a638d11d9254f2f3b50a4b2ce16b03e2db0ad2742fce155352c792cdeed1b244945430598d170eae546ab1e0cd4d4beedf5f0d9ea2c2
7
+ data.tar.gz: 6f54a08d03c458e7cf3dced5df532fbb5f5e2df2c9e8669c52a9bb3ebb0da231246717384653add9e585cbd2a068c00ab5be0fd12505fd04fccb7d0ca970faa3
@@ -0,0 +1,12 @@
1
+ name: Autocloser
2
+ on: [issues]
3
+ jobs:
4
+ autoclose:
5
+ runs-on: ubuntu-latest
6
+ steps:
7
+ - name: Autoclose issues that did not follow issue template
8
+ uses: roots/issue-closer-action@v1.1
9
+ with:
10
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
11
+ issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
12
+ issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"
@@ -0,0 +1,26 @@
1
+ name: Testing on Ubuntu
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.4', '2.5', '2.6' ]
12
+ os:
13
+ - ubuntu-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
@@ -0,0 +1,26 @@
1
+ name: Testing on macOS
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.4', '2.5', '2.6' ]
12
+ os:
13
+ - macOS-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
@@ -0,0 +1,26 @@
1
+ name: Testing on Windows
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ ruby: [ '2.4', '2.5', '2.6' ]
12
+ os:
13
+ - windows-latest
14
+ name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
+ steps:
16
+ - uses: actions/checkout@v2
17
+ - uses: ruby/setup-ruby@v1
18
+ with:
19
+ ruby-version: ${{ matrix.ruby }}
20
+ - name: unit testing
21
+ env:
22
+ CI: true
23
+ run: |
24
+ gem install bundler rake
25
+ bundle install --jobs 4 --retry 3
26
+ bundle exec rake test
data/History.md CHANGED
@@ -2,6 +2,31 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 4.1.0
6
+ - Implement Fallback selector and configurable selector class (#782)
7
+
8
+ ### 4.0.11
9
+ - Add custom and time placeholders combination testcase for ILM (#781)
10
+ - Really support ILM on `logstash_format` enabled environment (#779)
11
+
12
+ ### 4.0.10
13
+ - filter_elasticsearch_genid: Use entire record as hash seed (#777)
14
+ - Suppress type in meta with suppress_type_name parameter (#774)
15
+ - filter\_elasticsearch\_genid: Add hash generation mechanism from events (#773)
16
+ - Clean up error text (#772)
17
+ - Use GitHub Actions badges instead of Travis' (#760)
18
+ - Add issue auto closer workflow (#759)
19
+ - Document required permissions (#757)
20
+
21
+ ### 4.0.9
22
+ - Add possibility to configure multiple ILM policies (#753)
23
+ - Document required permissions (#757)
24
+
25
+ ### 4.0.8
26
+ - Handle compressable connection usable state (#743)
27
+ - Use newer tls protocol versions (#739)
28
+ - Add GitHub Actions file (#740)
29
+
5
30
  ### 4.0.7
6
31
  - Added http_backend_excon_nonblock config in out_elasticsearch (#733)
7
32
 
@@ -0,0 +1,116 @@
1
+ ## Index
2
+
3
+ * [Usage](#usage)
4
+ * [Configuration](#configuration)
5
+ + [hash_id_key](#hash_id_key)
6
+ + [include_tag_in_seed](#include_tag_in_seed)
7
+ + [include_time_in_seed](#include_time_in_seed)
8
+ + [use_record_as_seed](#use_record_as_seed)
9
+ + [use_entire_record](#use_entire_record)
10
+ + [record_keys](#record_keys)
11
+ + [separator](#separator)
12
+ + [hash_type](#hash_type)
13
+ * [Advanced Usage](#advanced-usage)
14
+
15
+ ## Usage
16
+
17
+ In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
18
+
19
+ ```
20
+ <source>
21
+ @type elasticsearch_genid
22
+ hash_id_key _hash
23
+ include_tag_in_seed false
24
+ include_time_in_seed false
25
+ use_record_as_seed false
26
+ use_entire_record false
27
+ record_keys []
28
+ separator _
29
+ hash_type md5
30
+ </match>
31
+ ```
32
+
33
+ ## Configuration
34
+
35
+ ### hash_id_key
36
+
37
+ ```
38
+ hash_id_key _id
39
+ ```
40
+
41
+ You can specify generated hash storing key.
42
+
43
+ ### include_tag_in_seed
44
+
45
+ ```
46
+ include_tag_in_seed true
47
+ ```
48
+
49
+ You can specify to use tag for hash generation seed.
50
+
51
+ ### include_time_in_seed
52
+
53
+ ```
54
+ include_time_in_seed true
55
+ ```
56
+
57
+ You can specify to use time for hash generation seed.
58
+
59
+ ### use_record_as_seed
60
+
61
+ ```
62
+ use_record_as_seed true
63
+ ```
64
+
65
+ You can specify to use record in events for hash generation seed. This parameter should be used with [record_keys](#record_keys) parameter in practice.
66
+
67
+ ### record_keys
68
+
69
+ ```
70
+ record_keys request_id,pipeline_id
71
+ ```
72
+
73
+ You can specify keys which are record in events for hash generation seed. This parameter should be used with [use_record_as_seed](#use_record_as_seed) parameter in practice.
74
+
75
+ ### use_entire_record
76
+
77
+ ```
78
+ use_entire_record true
79
+ ```
80
+
81
+ You can specify to use entire record in events for hash generation seed.
82
+
83
+
84
+ ### separator
85
+
86
+ ```
87
+ separator |
88
+ ```
89
+
90
+ You can specify separator charactor to creating seed for hash generation.
91
+
92
+ ### hash_type
93
+
94
+ ```
95
+ hash_type sha1
96
+ ```
97
+
98
+ You can specify hash algorithm.
99
+
100
+ ## Advanced Usage
101
+
102
+ Elasticsearch GenID plugin can handle record contents differing with the following parameters:
103
+
104
+ ```aconf
105
+ <filter the.awesome.your.routing.tag>
106
+ @type elasticsearch_genid
107
+ use_entire_record true
108
+ hash_type sha1
109
+ hash_id_key _hash
110
+ separator _
111
+ inc_time_as_key true
112
+ inc_tag_as_key true
113
+ </filter>
114
+ ```
115
+
116
+ The above configuration can handle tag, time, and record differing and generate different base64 encoded hash per record.
data/README.md CHANGED
@@ -1,7 +1,9 @@
1
1
  # Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
2
2
 
3
3
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
4
- [![Build Status](https://travis-ci.org/uken/fluent-plugin-elasticsearch.png?branch=master)](https://travis-ci.org/uken/fluent-plugin-elasticsearch)
4
+ ![Testing on Windows](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Windows/badge.svg?branch=master)
5
+ ![Testing on macOS](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20macOS/badge.svg?branch=master)
6
+ ![Testing on Ubuntu](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Ubuntu/badge.svg?branch=master)
5
7
  [![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
6
8
  [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
7
9
 
@@ -31,6 +33,7 @@ Current maintainers: @cosmo0920
31
33
  + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
32
34
  + [include_timestamp](#include_timestamp)
33
35
  + [utc_index](#utc_index)
36
+ + [suppress_type_name](#suppress_type_name)
34
37
  + [target_index_key](#target_index_key)
35
38
  + [target_type_key](#target_type_key)
36
39
  + [template_name](#template_name)
@@ -75,6 +78,7 @@ Current maintainers: @cosmo0920
75
78
  + [Hash flattening](#hash-flattening)
76
79
  + [Generate Hash ID](#generate-hash-id)
77
80
  + [sniffer_class_name](#sniffer-class-name)
81
+ + [selector_class_name](#selector-class-name)
78
82
  + [reload_after](#reload-after)
79
83
  + [validate_client_version](#validate-client-version)
80
84
  + [unrecoverable_error_types](#unrecoverable-error-types)
@@ -93,9 +97,12 @@ Current maintainers: @cosmo0920
93
97
  + [enable_ilm](#enable_ilm)
94
98
  + [ilm_policy_id](#ilm_policy_id)
95
99
  + [ilm_policy](#ilm_policy)
100
+ + [ilm_policies](#ilm_policies)
96
101
  + [ilm_policy_overwrite](#ilm_policy_overwrite)
97
102
  + [truncate_caches_interval](#truncate_caches_interval)
98
103
  * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
104
+ * [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
105
+ * [Elasticsearch permissions](#elasticsearch-permissions)
99
106
  * [Troubleshooting](#troubleshooting)
100
107
  + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
101
108
  + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -349,6 +356,20 @@ utc_index true
349
356
 
350
357
  By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
351
358
 
359
+ ### suppress_type_name
360
+
361
+ In Elasticsearch 7.x, Elasticsearch cluster complains the following types removal warnings:
362
+
363
+ ```json
364
+ {"type": "deprecation", "timestamp": "2020-07-03T08:02:20,830Z", "level": "WARN", "component": "o.e.d.a.b.BulkRequestParser", "cluster.name": "docker-cluster", "node.name": "70dd5c6b94c3", "message": "[types removal] Specifying types in bulk requests is deprecated.", "cluster.uuid": "NoJJmtzfTtSzSMv0peG8Wg", "node.id": "VQ-PteHmTVam2Pnbg7xWHw" }
365
+ ```
366
+
367
+ This can be suppressed with:
368
+
369
+ ```
370
+ suppress_type_name true
371
+ ```
372
+
352
373
  ### target_index_key
353
374
 
354
375
  Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
@@ -881,7 +902,11 @@ ssl_min_version TLSv1_2
881
902
 
882
903
  Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
883
904
 
884
- If they are *not* specified in the Elasticsearch plugin configuration, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
905
+ If they are *not* specified in the Elasticsearch plugin configuration, `ssl_max_version` and `ssl_min_version` is set up with:
906
+
907
+ In Elasticsearch plugin v4.0.8 or later with Ruby 2.5 or later environment, `ssl_max_version` should be `TLSv1_3` and `ssl_min_version` should be `TLSv1_2`.
908
+
909
+ From Elasticsearch plugin v4.0.4 to v4.0.7 with Ruby 2.5 or later environment, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
885
910
 
886
911
 
887
912
  ### Proxy Support
@@ -963,6 +988,18 @@ sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
963
988
  reload_after 100
964
989
  ```
965
990
 
991
+ ### Selector Class Name
992
+
993
+ The default selector used by the `Elasticsearch::Transport` class works well when Fluentd should round robin and random selector cases. This doesn't work well when Fluentd should fallback behavior.
994
+ The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
995
+
996
+ The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
997
+
998
+ ```
999
+ hosts exhausted-host:9201,normal-host:9200
1000
+ selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
1001
+ ```
1002
+
966
1003
  #### Tips
967
1004
 
968
1005
  The included sniffer class does not required `out_elasticsearch`.
@@ -979,7 +1016,7 @@ If you use Fluentd directly, you must pass the following lines as Fluentd comman
979
1016
 
980
1017
  ```
981
1018
  sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
982
- $ fluentd -r $sniffer" [AND YOUR OTHER OPTIONS]
1019
+ $ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
983
1020
  ```
984
1021
 
985
1022
  ### Reload After
@@ -1209,6 +1246,14 @@ Default value is `{}`.
1209
1246
 
1210
1247
  **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1211
1248
 
1249
+ ## ilm_policies
1250
+
1251
+ A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
1252
+
1253
+ Default value is `{}`.
1254
+
1255
+ **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1256
+
1212
1257
  ## ilm_policy_overwrite
1213
1258
 
1214
1259
  Specify whether overwriting ilm policy or not.
@@ -1229,6 +1274,36 @@ Default value is `nil`.
1229
1274
 
1230
1275
  See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
1231
1276
 
1277
+ ## Configuration - Elasticsearch Filter GenID
1278
+
1279
+ See [Elasticsearch Filter GenID document](README.ElasticsearchGenID.md)
1280
+
1281
+ ## Elasticsearch permissions
1282
+
1283
+ If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
1284
+
1285
+ The set of required permissions are the following:
1286
+
1287
+ ```json
1288
+ "cluster": ["manage_index_templates", "monitor", "manage_ilm"],
1289
+ "indices": [
1290
+ {
1291
+ "names": [ "*" ],
1292
+ "privileges": ["write","create","delete","create_index","manage","manage_ilm"]
1293
+ }
1294
+ ]
1295
+ ```
1296
+
1297
+ These permissions can be narrowed down by:
1298
+
1299
+ - Setting a more specific pattern for indices under the `names` field
1300
+ - Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
1301
+ - Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
1302
+ features in the plugin configuration
1303
+
1304
+ The list of privileges along with their description can be found in
1305
+ [security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
1306
+
1232
1307
  ## Troubleshooting
1233
1308
 
1234
1309
  ### Cannot send events to Elasticsearch
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.0.7'
6
+ s.version = '4.1.0'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -0,0 +1,9 @@
1
+ require 'elasticsearch/transport/transport/connections/selector'
2
+
3
+ class Fluent::Plugin::ElasticseatchFallbackSelector
4
+ include Elasticsearch::Transport::Transport::Connections::Selector::Base
5
+
6
+ def select(options={})
7
+ connections.first
8
+ end
9
+ end
@@ -10,7 +10,7 @@ module Fluent::Plugin
10
10
  [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
11
11
  end
12
12
 
13
- DEFAULT_VERSION = :TLSv1
13
+ DEFAULT_VERSION = :TLSv1_2
14
14
  METHODS_MAP = begin
15
15
  # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
16
16
  map = {
@@ -48,8 +48,8 @@ module Fluent::Plugin
48
48
  if USE_TLS_MINMAX_VERSION
49
49
  case
50
50
  when ssl_min_version.nil? && ssl_max_version.nil?
51
- ssl_min_version = METHODS_MAP[ssl_version]
52
- ssl_max_version = METHODS_MAP[ssl_version]
51
+ ssl_min_version = METHODS_MAP[:TLSv1_2]
52
+ ssl_max_version = METHODS_MAP[:TLSv1_3]
53
53
  when ssl_min_version && ssl_max_version.nil?
54
54
  raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
55
55
  when ssl_min_version.nil? && ssl_max_version