fluent-plugin-elasticsearch 4.2.0 → 4.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/History.md +5 -0
- data/README.md +49 -1
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +19 -0
- data/test/plugin/test_elasticsearch_tls.rb +8 -2
- data/test/plugin/test_out_elasticsearch.rb +41 -3
- data/test/plugin/test_out_elasticsearch_dynamic.rb +7 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9eb6b06e90f827162d669e94fb9a622c09f12b35e0a05f82c961ad870e3f9726
|
4
|
+
data.tar.gz: c7964330b33c36176e4b1238e4ba4f2d55e60e6ee6da4fec6799f25d943ab85f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd5d961731f5866a7c841f9cbeb11f8434868988862cb9929af4cfbe174a7ce65c38d125b91dc8952120be2da4b5ad08e89c127f9c65b9e38585185acccd1cdc
|
7
|
+
data.tar.gz: 0c15c5670b0a2a9eda90b6962768c8db7e510e2da4b2434e3cb841d2fa5b15cf52a8f65f35b4236ab01cb8721d640c690f3bf753a4f4f269b2c802d9f2d7a9b6
|
data/History.md
CHANGED
data/README.md
CHANGED
@@ -102,6 +102,9 @@ Current maintainers: @cosmo0920
|
|
102
102
|
+ [ilm_policy_overwrite](#ilm_policy_overwrite)
|
103
103
|
+ [truncate_caches_interval](#truncate_caches_interval)
|
104
104
|
+ [use_legacy_template](#use_legacy_template)
|
105
|
+
+ [metadata section](#metadata-section)
|
106
|
+
+ [include_chunk_id](#include_chunk_id)
|
107
|
+
+ [chunk_id_key](#chunk_id_key)
|
105
108
|
* [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
|
106
109
|
* [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
|
107
110
|
* [Elasticsearch permissions](#elasticsearch-permissions)
|
@@ -921,7 +924,7 @@ Starting with version 0.8.0, this gem uses excon, which supports proxy with envi
|
|
921
924
|
|
922
925
|
### Buffer options
|
923
926
|
|
924
|
-
`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/
|
927
|
+
`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/output#overview) and use `compat_parameters` plugin helper. It adds the following options:
|
925
928
|
|
926
929
|
```
|
927
930
|
buffer_type memory
|
@@ -1316,6 +1319,51 @@ Composable template documentation is [Put Index Template API | Elasticsearch Ref
|
|
1316
1319
|
|
1317
1320
|
Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
|
1318
1321
|
|
1322
|
+
## <metadata\> section
|
1323
|
+
|
1324
|
+
Users can specify whether including `chunk_id` information into records or not:
|
1325
|
+
|
1326
|
+
```aconf
|
1327
|
+
<match your.awesome.routing.tag>
|
1328
|
+
@type elasticsearch
|
1329
|
+
# Other configurations.
|
1330
|
+
<metadata>
|
1331
|
+
include_chunk_id true
|
1332
|
+
# chunk_id_key chunk_id # Default value is "chunk_id".
|
1333
|
+
</metadata>
|
1334
|
+
</match>
|
1335
|
+
```
|
1336
|
+
|
1337
|
+
### include_chunk_id
|
1338
|
+
|
1339
|
+
Whether including `chunk_id` for not. Default value is `false`.
|
1340
|
+
|
1341
|
+
```aconf
|
1342
|
+
<match your.awesome.routing.tag>
|
1343
|
+
@type elasticsearch
|
1344
|
+
# Other configurations.
|
1345
|
+
<metadata>
|
1346
|
+
include_chunk_id true
|
1347
|
+
</metadata>
|
1348
|
+
</match>
|
1349
|
+
```
|
1350
|
+
|
1351
|
+
|
1352
|
+
### chunk_id_key
|
1353
|
+
|
1354
|
+
Specify `chunk_id_key` to store `chunk_id` information into records. Default value is `chunk_id`.
|
1355
|
+
|
1356
|
+
```aconf
|
1357
|
+
<match your.awesome.routing.tag>
|
1358
|
+
@type elasticsearch
|
1359
|
+
# Other configurations.
|
1360
|
+
<metadata>
|
1361
|
+
include_chunk_id
|
1362
|
+
chunk_id_key chunk_hex
|
1363
|
+
</metadata>
|
1364
|
+
</match>
|
1365
|
+
```
|
1366
|
+
|
1319
1367
|
## Configuration - Elasticsearch Input
|
1320
1368
|
|
1321
1369
|
See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.2.
|
6
|
+
s.version = '4.2.1'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -18,6 +18,7 @@ require 'fluent/plugin/output'
|
|
18
18
|
require 'fluent/event'
|
19
19
|
require 'fluent/error'
|
20
20
|
require 'fluent/time'
|
21
|
+
require 'fluent/unique_id'
|
21
22
|
require 'fluent/log-ext'
|
22
23
|
require 'zlib'
|
23
24
|
require_relative 'elasticsearch_constants'
|
@@ -171,6 +172,11 @@ EOC
|
|
171
172
|
config_param :truncate_caches_interval, :time, :default => nil
|
172
173
|
config_param :use_legacy_template, :bool, :default => true
|
173
174
|
|
175
|
+
config_section :metadata, param_name: :metainfo, multi: false do
|
176
|
+
config_param :include_chunk_id, :bool, :default => false
|
177
|
+
config_param :chunk_id_key, :string, :default => "chunk_id".freeze
|
178
|
+
end
|
179
|
+
|
174
180
|
config_section :buffer do
|
175
181
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
176
182
|
config_set_default :chunk_keys, ['tag']
|
@@ -771,6 +777,15 @@ EOC
|
|
771
777
|
true
|
772
778
|
end
|
773
779
|
|
780
|
+
def inject_chunk_id_to_record_if_needed(record, chunk_id)
|
781
|
+
if @metainfo&.include_chunk_id
|
782
|
+
record[@metainfo.chunk_id_key] = chunk_id
|
783
|
+
record
|
784
|
+
else
|
785
|
+
record
|
786
|
+
end
|
787
|
+
end
|
788
|
+
|
774
789
|
def write(chunk)
|
775
790
|
bulk_message_count = Hash.new { |h,k| h[k] = 0 }
|
776
791
|
bulk_message = Hash.new { |h,k| h[k] = '' }
|
@@ -778,6 +793,7 @@ EOC
|
|
778
793
|
meta = {}
|
779
794
|
|
780
795
|
tag = chunk.metadata.tag
|
796
|
+
chunk_id = dump_unique_id_hex(chunk.unique_id)
|
781
797
|
extracted_values = expand_placeholders(chunk)
|
782
798
|
host = if @hosts
|
783
799
|
extract_placeholders(@hosts, chunk)
|
@@ -787,6 +803,9 @@ EOC
|
|
787
803
|
|
788
804
|
chunk.msgpack_each do |time, record|
|
789
805
|
next unless record.is_a? Hash
|
806
|
+
|
807
|
+
record = inject_chunk_id_to_record_if_needed(record, chunk_id)
|
808
|
+
|
790
809
|
begin
|
791
810
|
meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
|
792
811
|
info = if @include_index_in_url
|
@@ -62,8 +62,14 @@ class TestElasticsearchTLS < Test::Unit::TestCase
|
|
62
62
|
d = driver('')
|
63
63
|
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
64
64
|
if @use_tls_minmax_version
|
65
|
-
|
66
|
-
|
65
|
+
if @enabled_tlsv1_3
|
66
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
67
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
68
|
+
else
|
69
|
+
assert_equal({max_version: nil,
|
70
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
71
|
+
|
72
|
+
end
|
67
73
|
else
|
68
74
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
69
75
|
end
|
@@ -230,8 +230,13 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
230
230
|
assert_nil instance.ssl_max_version
|
231
231
|
assert_nil instance.ssl_min_version
|
232
232
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
233
|
-
|
234
|
-
|
233
|
+
if defined?(OpenSSL::SSL::TLS1_3_VERSION)
|
234
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
235
|
+
instance.ssl_version_options)
|
236
|
+
else
|
237
|
+
assert_equal({max_version: nil, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
238
|
+
instance.ssl_version_options)
|
239
|
+
end
|
235
240
|
else
|
236
241
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
237
242
|
instance.ssl_version_options)
|
@@ -1873,7 +1878,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1873
1878
|
else
|
1874
1879
|
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
|
1875
1880
|
with(basic_auth: ['john', 'doe'],
|
1876
|
-
body: "{\"index_patterns\":\"task_definition
|
1881
|
+
body: "{\"index_patterns\":\"task_definition-#{date_str}-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"task_definition-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
1877
1882
|
to_return(:status => 200, :body => "", :headers => {})
|
1878
1883
|
end
|
1879
1884
|
# check if alias exists
|
@@ -3404,6 +3409,39 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3404
3409
|
assert_requested(request, times: 2)
|
3405
3410
|
end
|
3406
3411
|
|
3412
|
+
def test_writes_with_record_metadata
|
3413
|
+
chunk_id_key = "metadata_key".freeze
|
3414
|
+
driver.configure(Fluent::Config::Element.new(
|
3415
|
+
'ROOT', '', {
|
3416
|
+
'@type' => 'elasticsearch',
|
3417
|
+
}, [
|
3418
|
+
Fluent::Config::Element.new('metadata', '', {
|
3419
|
+
'include_chunk_id' => true,
|
3420
|
+
'chunk_id_key' => chunk_id_key,
|
3421
|
+
}, [])
|
3422
|
+
]
|
3423
|
+
))
|
3424
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
3425
|
+
with(
|
3426
|
+
body: /{"index":{"_index":"fluentd","_type":"fluentd"}}\n{"age":26,"request_id":"42","parent_id":"parent","routing_id":"routing","#{chunk_id_key}":".*"}\n/) do |req|
|
3427
|
+
@index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
3428
|
+
end
|
3429
|
+
driver.run(default_tag: 'test', shutdown: false) do
|
3430
|
+
driver.feed(sample_record)
|
3431
|
+
end
|
3432
|
+
assert_true index_cmds[1].has_key?(chunk_id_key)
|
3433
|
+
first_chunk_id = index_cmds[1].fetch(chunk_id_key)
|
3434
|
+
|
3435
|
+
driver.run(default_tag: 'test') do
|
3436
|
+
driver.feed(sample_record)
|
3437
|
+
end
|
3438
|
+
assert_true index_cmds[1].has_key?(chunk_id_key)
|
3439
|
+
second_chunk_id = index_cmds[1].fetch(chunk_id_key)
|
3440
|
+
assert do
|
3441
|
+
first_chunk_id != second_chunk_id
|
3442
|
+
end
|
3443
|
+
end
|
3444
|
+
|
3407
3445
|
def test_writes_with_huge_records_but_uncheck
|
3408
3446
|
driver.configure(Fluent::Config::Element.new(
|
3409
3447
|
'ROOT', '', {
|
@@ -101,8 +101,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
101
101
|
assert_nil instance.ssl_max_version
|
102
102
|
assert_nil instance.ssl_min_version
|
103
103
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
104
|
-
|
105
|
-
|
104
|
+
if defined?(OpenSSL::SSL::TLS1_3_VERSION)
|
105
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
106
|
+
instance.ssl_version_options)
|
107
|
+
else
|
108
|
+
assert_equal({max_version: nil, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
109
|
+
instance.ssl_version_options)
|
110
|
+
end
|
106
111
|
else
|
107
112
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
108
113
|
instance.ssl_version_options)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.2.
|
4
|
+
version: 4.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-
|
13
|
+
date: 2020-10-12 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|