fluent-plugin-elasticsearch 4.0.10 → 4.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +4 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +8 -5
- data/test/plugin/test_out_elasticsearch.rb +153 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c90e0b347a5c90ec23231df927384dadcc4b91b813a4e2c11df23539dfe05628
|
4
|
+
data.tar.gz: d6e296c8006870a5f85bd0b97294c69d797fc3f40b574ede7645fcd808ae6356
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8466c4e3a13ef5c5ca1bda536a0baf7044fdaa0bc3ae1e1c7e6d0e6cf3dc7596d2b68e3f61ed2fb788f8ef2d8392939b8f63fd6aedc53e5818725bc6e0910aa6
|
7
|
+
data.tar.gz: b0572e50952ce49e44b280e94764d0d204d482e8e89cc2124e9a3025c1bd4cf7debe5625ad141337e5f81e6b6816286ee81f931f6b782f0a8cffcbfa0ed12dc3
|
data/History.md
CHANGED
@@ -2,6 +2,10 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 4.0.11
|
6
|
+
- Add custom and time placeholders combination testcase for ILM (#781)
|
7
|
+
- Really support ILM on `logstash_format` enabled environment (#779)
|
8
|
+
|
5
9
|
### 4.0.10
|
6
10
|
- filter_elasticsearch_genid: Use entire record as hash seed (#777)
|
7
11
|
- Suppress type in meta with suppress_type_name parameter (#774)
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.0.
|
6
|
+
s.version = '4.0.11'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -761,9 +761,9 @@ EOC
|
|
761
761
|
begin
|
762
762
|
meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
|
763
763
|
info = if @include_index_in_url
|
764
|
-
RequestInfo.new(host, meta.delete("_index".freeze), meta
|
764
|
+
RequestInfo.new(host, meta.delete("_index".freeze), meta.delete("_alias".freeze))
|
765
765
|
else
|
766
|
-
RequestInfo.new(host, nil, meta
|
766
|
+
RequestInfo.new(host, nil, meta.delete("_alias".freeze))
|
767
767
|
end
|
768
768
|
|
769
769
|
if split_request?(bulk_message, info)
|
@@ -809,7 +809,7 @@ EOC
|
|
809
809
|
end
|
810
810
|
|
811
811
|
def process_message(tag, meta, header, time, record, extracted_values)
|
812
|
-
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias,
|
812
|
+
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
|
813
813
|
|
814
814
|
if @flatten_hashes
|
815
815
|
record = flatten_record(record)
|
@@ -832,17 +832,19 @@ EOC
|
|
832
832
|
|
833
833
|
target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
|
834
834
|
if target_index_parent && target_index_parent[target_index_child_key]
|
835
|
-
target_index = target_index_parent.delete(target_index_child_key)
|
835
|
+
target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
|
836
836
|
elsif @logstash_format
|
837
837
|
dt = dt.new_offset(0) if @utc_index
|
838
838
|
target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
|
839
|
+
target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
|
839
840
|
else
|
840
|
-
target_index = index_name
|
841
|
+
target_index_alias = target_index = index_name
|
841
842
|
end
|
842
843
|
|
843
844
|
# Change target_index to lower-case since Elasticsearch doesn't
|
844
845
|
# allow upper-case characters in index names.
|
845
846
|
target_index = target_index.downcase
|
847
|
+
target_index_alias = target_index_alias.downcase
|
846
848
|
if @include_tag_key
|
847
849
|
record[@tag_key] = tag
|
848
850
|
end
|
@@ -877,6 +879,7 @@ EOC
|
|
877
879
|
meta.clear
|
878
880
|
meta["_index".freeze] = target_index
|
879
881
|
meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
|
882
|
+
meta["_alias".freeze] = target_index_alias
|
880
883
|
|
881
884
|
if @pipeline
|
882
885
|
meta["pipeline".freeze] = pipeline
|
@@ -1053,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1053
1053
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1054
1054
|
end
|
1055
1055
|
|
1056
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
|
1057
|
+
cwd = File.dirname(__FILE__)
|
1058
|
+
template_file = File.join(cwd, 'test_template.json')
|
1059
|
+
|
1060
|
+
config = %{
|
1061
|
+
host logs.google.com
|
1062
|
+
port 777
|
1063
|
+
scheme https
|
1064
|
+
path /es/
|
1065
|
+
user john
|
1066
|
+
password doe
|
1067
|
+
template_name logstash
|
1068
|
+
template_file #{template_file}
|
1069
|
+
index_date_pattern now/w{xxxx.ww}
|
1070
|
+
enable_ilm true
|
1071
|
+
logstash_format true
|
1072
|
+
application_name log
|
1073
|
+
}
|
1074
|
+
|
1075
|
+
date_str = Time.now.strftime("%Y.%m.%d")
|
1076
|
+
# connection start
|
1077
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1078
|
+
with(basic_auth: ['john', 'doe']).
|
1079
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1080
|
+
# check if template exists
|
1081
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1082
|
+
with(basic_auth: ['john', 'doe']).
|
1083
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1084
|
+
# creation
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1086
|
+
with(basic_auth: ['john', 'doe']).
|
1087
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1088
|
+
# check if alias exists
|
1089
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-log-#{date_str}").
|
1090
|
+
with(basic_auth: ['john', 'doe']).
|
1091
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1092
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1093
|
+
with(basic_auth: ['john', 'doe']).
|
1094
|
+
to_return(status: 404, body: "", headers: {})
|
1095
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1096
|
+
with(basic_auth: ['john', 'doe'],
|
1097
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
1098
|
+
to_return(status: 200, body: "", headers: {})
|
1099
|
+
# put the alias for the index
|
1100
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1101
|
+
with(basic_auth: ['john', 'doe']).
|
1102
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1103
|
+
|
1104
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-log-#{date_str}").
|
1105
|
+
with(basic_auth: ['john', 'doe'],
|
1106
|
+
body: "{\"aliases\":{\"logstash-log-#{date_str}\":{\"is_write_index\":true}}}").
|
1107
|
+
to_return(status: 200, body: "", headers: {})
|
1108
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1109
|
+
with(basic_auth: ['john', 'doe']).
|
1110
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1111
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1112
|
+
with(basic_auth: ['john', 'doe']).
|
1113
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1114
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1115
|
+
with(basic_auth: ['john', 'doe'],
|
1116
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
1117
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1118
|
+
|
1119
|
+
driver(config)
|
1120
|
+
|
1121
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1122
|
+
driver.run(default_tag: 'test') do
|
1123
|
+
driver.feed(sample_record)
|
1124
|
+
end
|
1125
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}", times: 1)
|
1126
|
+
|
1127
|
+
assert_requested(elastic_request)
|
1128
|
+
end
|
1129
|
+
|
1056
1130
|
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
|
1057
1131
|
cwd = File.dirname(__FILE__)
|
1058
1132
|
template_file = File.join(cwd, 'test_template.json')
|
@@ -1607,6 +1681,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1607
1681
|
|
1608
1682
|
assert_requested(elastic_request)
|
1609
1683
|
end
|
1684
|
+
|
1685
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
|
1686
|
+
cwd = File.dirname(__FILE__)
|
1687
|
+
template_file = File.join(cwd, 'test_template.json')
|
1688
|
+
|
1689
|
+
config = Fluent::Config::Element.new(
|
1690
|
+
'ROOT', '', {
|
1691
|
+
'@type' => 'elasticsearch',
|
1692
|
+
'host' => 'logs.google.com',
|
1693
|
+
'port' => 777,
|
1694
|
+
'scheme' => "https",
|
1695
|
+
'path' => "/es/",
|
1696
|
+
'user' => 'john',
|
1697
|
+
'password' => 'doe',
|
1698
|
+
'template_name' => 'logstash',
|
1699
|
+
'template_file' => "#{template_file}",
|
1700
|
+
'index_date_pattern' => 'now/w{xxxx.ww}',
|
1701
|
+
'index_name' => "${taskDef}-%Y.%m",
|
1702
|
+
'enable_ilm' => true,
|
1703
|
+
}, [
|
1704
|
+
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
1705
|
+
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
1706
|
+
'timekey' => 3600,
|
1707
|
+
}, [])
|
1708
|
+
]
|
1709
|
+
)
|
1710
|
+
|
1711
|
+
task_def_value = "task_definition"
|
1712
|
+
date_str = Time.now.strftime("%Y.%m")
|
1713
|
+
# connection start
|
1714
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1715
|
+
with(basic_auth: ['john', 'doe']).
|
1716
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1717
|
+
# check if template exists
|
1718
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
1719
|
+
with(basic_auth: ['john', 'doe']).
|
1720
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1721
|
+
# creation
|
1722
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
1723
|
+
with(basic_auth: ['john', 'doe'],
|
1724
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
1725
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1726
|
+
# check if alias exists
|
1727
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
1728
|
+
with(basic_auth: ['john', 'doe']).
|
1729
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1730
|
+
# put the alias for the index
|
1731
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1732
|
+
with(basic_auth: ['john', 'doe']).
|
1733
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1734
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
|
1735
|
+
with(basic_auth: ['john', 'doe'],
|
1736
|
+
:body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
|
1737
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1738
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1739
|
+
with(basic_auth: ['john', 'doe']).
|
1740
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1741
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1742
|
+
with(basic_auth: ['john', 'doe']).
|
1743
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1744
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1745
|
+
with(basic_auth: ['john', 'doe'],
|
1746
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
1747
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1748
|
+
|
1749
|
+
driver(config)
|
1750
|
+
|
1751
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1752
|
+
driver.run(default_tag: 'test') do
|
1753
|
+
driver.feed(sample_record.merge("taskDef" => task_def_value))
|
1754
|
+
end
|
1755
|
+
assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
|
1756
|
+
|
1757
|
+
assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
|
1758
|
+
|
1759
|
+
assert_requested(elastic_request)
|
1760
|
+
end
|
1610
1761
|
end
|
1611
1762
|
|
1612
1763
|
def test_custom_template_create
|
@@ -1863,11 +2014,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1863
2014
|
to_return(:status => 200, :body => "", :headers => {})
|
1864
2015
|
# check if alias exists
|
1865
2016
|
timestr = Time.now.strftime("%Y.%m.%d")
|
1866
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
2017
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
|
1867
2018
|
with(basic_auth: ['john', 'doe']).
|
1868
2019
|
to_return(:status => 404, :body => "", :headers => {})
|
1869
2020
|
# put the alias for the index
|
1870
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
2021
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-myapp-#{timestr}").
|
1871
2022
|
with(basic_auth: ['john', 'doe']).
|
1872
2023
|
to_return(:status => 200, :body => "", :headers => {})
|
1873
2024
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.
|
4
|
+
version: 4.0.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-07-
|
13
|
+
date: 2020-07-09 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|