fluent-plugin-elasticsearch 4.0.9 → 4.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/History.md +22 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +74 -9
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/elasticsearch_index_template.rb +19 -11
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +48 -25
- data/test/plugin/test_elasticsearch_fallback_selector.rb +73 -0
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +171 -80
- metadata +10 -5
@@ -707,7 +707,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
707
707
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
708
708
|
def detect_es_major_version
|
709
709
|
@_es_info ||= client.info
|
710
|
-
@_es_info
|
710
|
+
unless version = @_es_info.dig("version", "number")
|
711
|
+
version = @default_elasticsearch_version
|
712
|
+
end
|
713
|
+
version.to_i
|
711
714
|
end
|
712
715
|
CODE
|
713
716
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
@@ -753,7 +756,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
753
756
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
754
757
|
def detect_es_major_version
|
755
758
|
@_es_info ||= client.info
|
756
|
-
@_es_info
|
759
|
+
unless version = @_es_info.dig("version", "number")
|
760
|
+
version = @default_elasticsearch_version
|
761
|
+
end
|
762
|
+
version.to_i
|
757
763
|
end
|
758
764
|
CODE
|
759
765
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
@@ -1053,7 +1059,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1053
1059
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1054
1060
|
end
|
1055
1061
|
|
1056
|
-
def
|
1062
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
|
1057
1063
|
cwd = File.dirname(__FILE__)
|
1058
1064
|
template_file = File.join(cwd, 'test_template.json')
|
1059
1065
|
|
@@ -1067,60 +1073,67 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1067
1073
|
template_name logstash
|
1068
1074
|
template_file #{template_file}
|
1069
1075
|
index_date_pattern now/w{xxxx.ww}
|
1070
|
-
index_name logstash
|
1071
1076
|
enable_ilm true
|
1072
|
-
|
1073
|
-
|
1077
|
+
logstash_format true
|
1078
|
+
application_name log
|
1074
1079
|
}
|
1075
1080
|
|
1081
|
+
date_str = Time.now.strftime("%Y.%m.%d")
|
1076
1082
|
# connection start
|
1077
1083
|
stub_request(:head, "https://logs.google.com:777/es//").
|
1078
1084
|
with(basic_auth: ['john', 'doe']).
|
1079
1085
|
to_return(:status => 200, :body => "", :headers => {})
|
1080
1086
|
# check if template exists
|
1081
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1087
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
1082
1088
|
with(basic_auth: ['john', 'doe']).
|
1083
1089
|
to_return(:status => 404, :body => "", :headers => {})
|
1084
1090
|
# creation
|
1085
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1091
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
1086
1092
|
with(basic_auth: ['john', 'doe']).
|
1087
1093
|
to_return(:status => 200, :body => "", :headers => {})
|
1088
1094
|
# check if alias exists
|
1089
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1095
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-#{date_str}").
|
1090
1096
|
with(basic_auth: ['john', 'doe']).
|
1091
1097
|
to_return(:status => 404, :body => "", :headers => {})
|
1092
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1098
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
1093
1099
|
with(basic_auth: ['john', 'doe']).
|
1094
1100
|
to_return(status: 404, body: "", headers: {})
|
1095
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1101
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
1096
1102
|
with(basic_auth: ['john', 'doe'],
|
1097
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":
|
1103
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
1098
1104
|
to_return(status: 200, body: "", headers: {})
|
1099
1105
|
# put the alias for the index
|
1100
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-
|
1106
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E").
|
1101
1107
|
with(basic_auth: ['john', 'doe']).
|
1102
1108
|
to_return(:status => 200, :body => "", :headers => {})
|
1103
|
-
|
1109
|
+
|
1110
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E/#{alias_endpoint}/logstash-#{date_str}").
|
1104
1111
|
with(basic_auth: ['john', 'doe'],
|
1105
|
-
:
|
1106
|
-
to_return(:
|
1112
|
+
body: "{\"aliases\":{\"logstash-#{date_str}\":{\"is_write_index\":true}}}").
|
1113
|
+
to_return(status: 200, body: "", headers: {})
|
1107
1114
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1108
1115
|
with(basic_auth: ['john', 'doe']).
|
1109
1116
|
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1110
1117
|
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1111
1118
|
with(basic_auth: ['john', 'doe']).
|
1112
|
-
to_return(:status =>
|
1119
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1113
1120
|
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1114
1121
|
with(basic_auth: ['john', 'doe'],
|
1115
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"
|
1122
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
1116
1123
|
to_return(:status => 200, :body => "", :headers => {})
|
1117
1124
|
|
1118
1125
|
driver(config)
|
1119
1126
|
|
1120
|
-
|
1127
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1128
|
+
driver.run(default_tag: 'test') do
|
1129
|
+
driver.feed(sample_record)
|
1130
|
+
end
|
1131
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}", times: 1)
|
1132
|
+
|
1133
|
+
assert_requested(elastic_request)
|
1121
1134
|
end
|
1122
1135
|
|
1123
|
-
def
|
1136
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
|
1124
1137
|
cwd = File.dirname(__FILE__)
|
1125
1138
|
template_file = File.join(cwd, 'test_template.json')
|
1126
1139
|
|
@@ -1134,9 +1147,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1134
1147
|
template_name logstash
|
1135
1148
|
template_file #{template_file}
|
1136
1149
|
index_date_pattern now/w{xxxx.ww}
|
1137
|
-
deflector_alias myapp_deflector
|
1138
1150
|
index_name logstash
|
1139
1151
|
enable_ilm true
|
1152
|
+
ilm_policy_overwrite true
|
1153
|
+
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
|
1140
1154
|
}
|
1141
1155
|
|
1142
1156
|
# connection start
|
@@ -1152,38 +1166,63 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1152
1166
|
with(basic_auth: ['john', 'doe']).
|
1153
1167
|
to_return(:status => 200, :body => "", :headers => {})
|
1154
1168
|
# check if alias exists
|
1155
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/
|
1169
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1156
1170
|
with(basic_auth: ['john', 'doe']).
|
1157
1171
|
to_return(:status => 404, :body => "", :headers => {})
|
1158
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/
|
1172
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1159
1173
|
with(basic_auth: ['john', 'doe']).
|
1160
1174
|
to_return(status: 404, body: "", headers: {})
|
1161
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/
|
1175
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1162
1176
|
with(basic_auth: ['john', 'doe'],
|
1163
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"
|
1177
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1164
1178
|
to_return(status: 200, body: "", headers: {})
|
1165
1179
|
# put the alias for the index
|
1166
1180
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1167
1181
|
with(basic_auth: ['john', 'doe']).
|
1168
1182
|
to_return(:status => 200, :body => "", :headers => {})
|
1169
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/
|
1183
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1170
1184
|
with(basic_auth: ['john', 'doe'],
|
1171
|
-
:body => "{\"aliases\":{\"
|
1185
|
+
:body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1172
1186
|
to_return(:status => 200, :body => "", :headers => {})
|
1173
1187
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1174
1188
|
with(basic_auth: ['john', 'doe']).
|
1175
1189
|
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1176
1190
|
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1177
1191
|
with(basic_auth: ['john', 'doe']).
|
1178
|
-
to_return(:status =>
|
1192
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1179
1193
|
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1180
1194
|
with(basic_auth: ['john', 'doe'],
|
1181
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"
|
1195
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"60gb\",\"max_age\":\"45d\"}}}}}}").
|
1182
1196
|
to_return(:status => 200, :body => "", :headers => {})
|
1183
1197
|
|
1184
1198
|
driver(config)
|
1185
1199
|
|
1186
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/
|
1200
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1201
|
+
end
|
1202
|
+
|
1203
|
+
def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
|
1204
|
+
cwd = File.dirname(__FILE__)
|
1205
|
+
template_file = File.join(cwd, 'test_template.json')
|
1206
|
+
|
1207
|
+
config = %{
|
1208
|
+
host logs.google.com
|
1209
|
+
port 777
|
1210
|
+
scheme https
|
1211
|
+
path /es/
|
1212
|
+
user john
|
1213
|
+
password doe
|
1214
|
+
template_name logstash
|
1215
|
+
template_file #{template_file}
|
1216
|
+
index_date_pattern now/w{xxxx.ww}
|
1217
|
+
deflector_alias myapp_deflector
|
1218
|
+
index_name logstash
|
1219
|
+
enable_ilm true
|
1220
|
+
}
|
1221
|
+
|
1222
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
1223
|
+
assert_raise(Fluent::ConfigError) do
|
1224
|
+
driver(config)
|
1225
|
+
end
|
1187
1226
|
end
|
1188
1227
|
|
1189
1228
|
def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
|
@@ -1607,6 +1646,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1607
1646
|
|
1608
1647
|
assert_requested(elastic_request)
|
1609
1648
|
end
|
1649
|
+
|
1650
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
|
1651
|
+
cwd = File.dirname(__FILE__)
|
1652
|
+
template_file = File.join(cwd, 'test_template.json')
|
1653
|
+
|
1654
|
+
config = Fluent::Config::Element.new(
|
1655
|
+
'ROOT', '', {
|
1656
|
+
'@type' => 'elasticsearch',
|
1657
|
+
'host' => 'logs.google.com',
|
1658
|
+
'port' => 777,
|
1659
|
+
'scheme' => "https",
|
1660
|
+
'path' => "/es/",
|
1661
|
+
'user' => 'john',
|
1662
|
+
'password' => 'doe',
|
1663
|
+
'template_name' => 'logstash',
|
1664
|
+
'template_file' => "#{template_file}",
|
1665
|
+
'index_date_pattern' => 'now/w{xxxx.ww}',
|
1666
|
+
'index_name' => "${taskDef}-%Y.%m",
|
1667
|
+
'enable_ilm' => true,
|
1668
|
+
}, [
|
1669
|
+
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
1670
|
+
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
1671
|
+
'timekey' => 3600,
|
1672
|
+
}, [])
|
1673
|
+
]
|
1674
|
+
)
|
1675
|
+
|
1676
|
+
task_def_value = "task_definition"
|
1677
|
+
date_str = Time.now.strftime("%Y.%m")
|
1678
|
+
# connection start
|
1679
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1680
|
+
with(basic_auth: ['john', 'doe']).
|
1681
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1682
|
+
# check if template exists
|
1683
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
1684
|
+
with(basic_auth: ['john', 'doe']).
|
1685
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1686
|
+
# creation
|
1687
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
1688
|
+
with(basic_auth: ['john', 'doe'],
|
1689
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
1690
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1691
|
+
# check if alias exists
|
1692
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
1693
|
+
with(basic_auth: ['john', 'doe']).
|
1694
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1695
|
+
# put the alias for the index
|
1696
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1697
|
+
with(basic_auth: ['john', 'doe']).
|
1698
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1699
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
|
1700
|
+
with(basic_auth: ['john', 'doe'],
|
1701
|
+
:body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
|
1702
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1703
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1704
|
+
with(basic_auth: ['john', 'doe']).
|
1705
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1706
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1707
|
+
with(basic_auth: ['john', 'doe']).
|
1708
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1709
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1710
|
+
with(basic_auth: ['john', 'doe'],
|
1711
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
1712
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1713
|
+
|
1714
|
+
driver(config)
|
1715
|
+
|
1716
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1717
|
+
driver.run(default_tag: 'test') do
|
1718
|
+
driver.feed(sample_record.merge("taskDef" => task_def_value))
|
1719
|
+
end
|
1720
|
+
assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
|
1721
|
+
|
1722
|
+
assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
|
1723
|
+
|
1724
|
+
assert_requested(elastic_request)
|
1725
|
+
end
|
1610
1726
|
end
|
1611
1727
|
|
1612
1728
|
def test_custom_template_create
|
@@ -1845,6 +1961,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1845
1961
|
application_name myapp
|
1846
1962
|
}
|
1847
1963
|
|
1964
|
+
timestr = Time.now.strftime("%Y.%m.%d")
|
1848
1965
|
# connection start
|
1849
1966
|
stub_request(:head, "https://logs.google.com:777/es//").
|
1850
1967
|
with(basic_auth: ['john', 'doe']).
|
@@ -1858,17 +1975,17 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1858
1975
|
with(basic_auth: ['john', 'doe']).
|
1859
1976
|
to_return(:status => 200, :body => "", :headers => {})
|
1860
1977
|
# creation of index which can rollover
|
1861
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
1978
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E").
|
1862
1979
|
with(basic_auth: ['john', 'doe']).
|
1863
1980
|
to_return(:status => 200, :body => "", :headers => {})
|
1864
1981
|
# check if alias exists
|
1865
|
-
timestr = Time.now.strftime("%Y.%m.%d")
|
1866
1982
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
1867
1983
|
with(basic_auth: ['john', 'doe']).
|
1868
1984
|
to_return(:status => 404, :body => "", :headers => {})
|
1869
1985
|
# put the alias for the index
|
1870
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
1871
|
-
with(basic_auth: ['john', 'doe']
|
1986
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
1987
|
+
with(basic_auth: ['john', 'doe'],
|
1988
|
+
body: "{\"aliases\":{\"mylogs-#{timestr}\":{\"is_write_index\":true}}}").
|
1872
1989
|
to_return(:status => 200, :body => "", :headers => {})
|
1873
1990
|
|
1874
1991
|
driver(config)
|
@@ -2051,52 +2168,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2051
2168
|
enable_ilm true
|
2052
2169
|
}
|
2053
2170
|
|
2054
|
-
#
|
2055
|
-
|
2056
|
-
|
2057
|
-
|
2058
|
-
# check if template exists
|
2059
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
2060
|
-
with(basic_auth: ['john', 'doe']).
|
2061
|
-
to_return(:status => 404, :body => "", :headers => {})
|
2062
|
-
# creation
|
2063
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
2064
|
-
with(basic_auth: ['john', 'doe']).
|
2065
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2066
|
-
# creation of index which can rollover
|
2067
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
2068
|
-
with(basic_auth: ['john', 'doe']).
|
2069
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2070
|
-
# check if alias exists
|
2071
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
2072
|
-
with(basic_auth: ['john', 'doe']).
|
2073
|
-
to_return(:status => 404, :body => "", :headers => {})
|
2074
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
2075
|
-
with(basic_auth: ['john', 'doe']).
|
2076
|
-
to_return(status: 404, body: "", headers: {})
|
2077
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
2078
|
-
with(basic_auth: ['john', 'doe'],
|
2079
|
-
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
|
2080
|
-
to_return(status: 200, body: "", headers: {})
|
2081
|
-
# put the alias for the index
|
2082
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
2083
|
-
with(basic_auth: ['john', 'doe'],
|
2084
|
-
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
2085
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2086
|
-
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
2087
|
-
with(basic_auth: ['john', 'doe']).
|
2088
|
-
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
2089
|
-
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
2090
|
-
with(basic_auth: ['john', 'doe']).
|
2091
|
-
to_return(:status => 404, :body => "", :headers => {})
|
2092
|
-
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
2093
|
-
with(basic_auth: ['john', 'doe'],
|
2094
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
2095
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2096
|
-
|
2097
|
-
driver(config)
|
2098
|
-
|
2099
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
|
2171
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
2172
|
+
assert_raise(Fluent::ConfigError) do
|
2173
|
+
driver(config)
|
2174
|
+
end
|
2100
2175
|
end
|
2101
2176
|
|
2102
2177
|
def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
|
@@ -3147,6 +3222,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3147
3222
|
|
3148
3223
|
data("border" => {"es_version" => 6, "_type" => "mytype"},
|
3149
3224
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
3225
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
3150
3226
|
)
|
3151
3227
|
def test_writes_to_speficied_type(data)
|
3152
3228
|
driver('', data["es_version"]).configure("type_name mytype\n")
|
@@ -3159,6 +3235,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3159
3235
|
|
3160
3236
|
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
3161
3237
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
3238
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
3162
3239
|
)
|
3163
3240
|
def test_writes_to_speficied_type_with_placeholders(data)
|
3164
3241
|
driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
|
@@ -3169,6 +3246,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3169
3246
|
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
3170
3247
|
end
|
3171
3248
|
|
3249
|
+
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
3250
|
+
"fixed_behavior"=> {"es_version" => 7, "_type" => nil},
|
3251
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
3252
|
+
)
|
3253
|
+
def test_writes_to_speficied_type_with_suppress_type_name(data)
|
3254
|
+
driver('', data["es_version"])
|
3255
|
+
.configure("type_name mytype.${tag}\nsuppress_type_name true")
|
3256
|
+
stub_elastic
|
3257
|
+
driver.run(default_tag: 'test') do
|
3258
|
+
driver.feed(sample_record)
|
3259
|
+
end
|
3260
|
+
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
3261
|
+
end
|
3262
|
+
|
3172
3263
|
data("old" => {"es_version" => 2, "_type" => "local-override"},
|
3173
3264
|
"old_behavior" => {"es_version" => 5, "_type" => "local-override"},
|
3174
3265
|
"border" => {"es_version" => 6, "_type" => "fluentd"},
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
8
8
|
- pitr
|
9
9
|
- Hiroshi Hatake
|
10
|
-
autorequire:
|
10
|
+
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-
|
13
|
+
date: 2020-08-19 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -135,6 +135,7 @@ extra_rdoc_files: []
|
|
135
135
|
files:
|
136
136
|
- ".coveralls.yml"
|
137
137
|
- ".editorconfig"
|
138
|
+
- ".github/workflows/issue-auto-closer.yml"
|
138
139
|
- ".github/workflows/linux.yml"
|
139
140
|
- ".github/workflows/macos.yml"
|
140
141
|
- ".github/workflows/windows.yml"
|
@@ -146,6 +147,7 @@ files:
|
|
146
147
|
- ISSUE_TEMPLATE.md
|
147
148
|
- LICENSE.txt
|
148
149
|
- PULL_REQUEST_TEMPLATE.md
|
150
|
+
- README.ElasticsearchGenID.md
|
149
151
|
- README.ElasticsearchInput.md
|
150
152
|
- README.md
|
151
153
|
- Rakefile
|
@@ -158,6 +160,7 @@ files:
|
|
158
160
|
- lib/fluent/plugin/elasticsearch_constants.rb
|
159
161
|
- lib/fluent/plugin/elasticsearch_error.rb
|
160
162
|
- lib/fluent/plugin/elasticsearch_error_handler.rb
|
163
|
+
- lib/fluent/plugin/elasticsearch_fallback_selector.rb
|
161
164
|
- lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb
|
162
165
|
- lib/fluent/plugin/elasticsearch_index_template.rb
|
163
166
|
- lib/fluent/plugin/elasticsearch_simple_sniffer.rb
|
@@ -170,6 +173,7 @@ files:
|
|
170
173
|
- test/helper.rb
|
171
174
|
- test/plugin/test_alias_template.json
|
172
175
|
- test/plugin/test_elasticsearch_error_handler.rb
|
176
|
+
- test/plugin/test_elasticsearch_fallback_selector.rb
|
173
177
|
- test/plugin/test_elasticsearch_index_lifecycle_management.rb
|
174
178
|
- test/plugin/test_elasticsearch_tls.rb
|
175
179
|
- test/plugin/test_filter_elasticsearch_genid.rb
|
@@ -184,7 +188,7 @@ licenses:
|
|
184
188
|
- Apache-2.0
|
185
189
|
metadata:
|
186
190
|
changelog_uri: https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md
|
187
|
-
post_install_message:
|
191
|
+
post_install_message:
|
188
192
|
rdoc_options: []
|
189
193
|
require_paths:
|
190
194
|
- lib
|
@@ -200,13 +204,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
200
204
|
version: '0'
|
201
205
|
requirements: []
|
202
206
|
rubygems_version: 3.1.2
|
203
|
-
signing_key:
|
207
|
+
signing_key:
|
204
208
|
specification_version: 4
|
205
209
|
summary: Elasticsearch output plugin for Fluent event collector
|
206
210
|
test_files:
|
207
211
|
- test/helper.rb
|
208
212
|
- test/plugin/test_alias_template.json
|
209
213
|
- test/plugin/test_elasticsearch_error_handler.rb
|
214
|
+
- test/plugin/test_elasticsearch_fallback_selector.rb
|
210
215
|
- test/plugin/test_elasticsearch_index_lifecycle_management.rb
|
211
216
|
- test/plugin/test_elasticsearch_tls.rb
|
212
217
|
- test/plugin/test_filter_elasticsearch_genid.rb
|