fluent-plugin-elasticsearch 4.0.9 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -707,7 +707,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
707
707
  Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
708
708
  def detect_es_major_version
709
709
  @_es_info ||= client.info
710
- @_es_info["version"]["number"].to_i
710
+ unless version = @_es_info.dig("version", "number")
711
+ version = @default_elasticsearch_version
712
+ end
713
+ version.to_i
711
714
  end
712
715
  CODE
713
716
  Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
@@ -753,7 +756,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
753
756
  Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
754
757
  def detect_es_major_version
755
758
  @_es_info ||= client.info
756
- @_es_info["version"]["number"].to_i
759
+ unless version = @_es_info.dig("version", "number")
760
+ version = @default_elasticsearch_version
761
+ end
762
+ version.to_i
757
763
  end
758
764
  CODE
759
765
  Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
@@ -1053,7 +1059,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1053
1059
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1054
1060
  end
1055
1061
 
1056
- def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
1062
+ def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
1057
1063
  cwd = File.dirname(__FILE__)
1058
1064
  template_file = File.join(cwd, 'test_template.json')
1059
1065
 
@@ -1067,60 +1073,67 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1067
1073
  template_name logstash
1068
1074
  template_file #{template_file}
1069
1075
  index_date_pattern now/w{xxxx.ww}
1070
- index_name logstash
1071
1076
  enable_ilm true
1072
- ilm_policy_overwrite true
1073
- ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
1077
+ logstash_format true
1078
+ application_name log
1074
1079
  }
1075
1080
 
1081
+ date_str = Time.now.strftime("%Y.%m.%d")
1076
1082
  # connection start
1077
1083
  stub_request(:head, "https://logs.google.com:777/es//").
1078
1084
  with(basic_auth: ['john', 'doe']).
1079
1085
  to_return(:status => 200, :body => "", :headers => {})
1080
1086
  # check if template exists
1081
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1087
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1082
1088
  with(basic_auth: ['john', 'doe']).
1083
1089
  to_return(:status => 404, :body => "", :headers => {})
1084
1090
  # creation
1085
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1091
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1086
1092
  with(basic_auth: ['john', 'doe']).
1087
1093
  to_return(:status => 200, :body => "", :headers => {})
1088
1094
  # check if alias exists
1089
- stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1095
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-#{date_str}").
1090
1096
  with(basic_auth: ['john', 'doe']).
1091
1097
  to_return(:status => 404, :body => "", :headers => {})
1092
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1098
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1093
1099
  with(basic_auth: ['john', 'doe']).
1094
1100
  to_return(status: 404, body: "", headers: {})
1095
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1101
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1096
1102
  with(basic_auth: ['john', 'doe'],
1097
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1103
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
1098
1104
  to_return(status: 200, body: "", headers: {})
1099
1105
  # put the alias for the index
1100
- stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1106
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E").
1101
1107
  with(basic_auth: ['john', 'doe']).
1102
1108
  to_return(:status => 200, :body => "", :headers => {})
1103
- stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1109
+
1110
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E/#{alias_endpoint}/logstash-#{date_str}").
1104
1111
  with(basic_auth: ['john', 'doe'],
1105
- :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1106
- to_return(:status => 200, :body => "", :headers => {})
1112
+ body: "{\"aliases\":{\"logstash-#{date_str}\":{\"is_write_index\":true}}}").
1113
+ to_return(status: 200, body: "", headers: {})
1107
1114
  stub_request(:get, "https://logs.google.com:777/es//_xpack").
1108
1115
  with(basic_auth: ['john', 'doe']).
1109
1116
  to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1110
1117
  stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1111
1118
  with(basic_auth: ['john', 'doe']).
1112
- to_return(:status => 200, :body => "", :headers => {})
1119
+ to_return(:status => 404, :body => "", :headers => {})
1113
1120
  stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1114
1121
  with(basic_auth: ['john', 'doe'],
1115
- :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"60gb\",\"max_age\":\"45d\"}}}}}}").
1122
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1116
1123
  to_return(:status => 200, :body => "", :headers => {})
1117
1124
 
1118
1125
  driver(config)
1119
1126
 
1120
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1127
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1128
+ driver.run(default_tag: 'test') do
1129
+ driver.feed(sample_record)
1130
+ end
1131
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}", times: 1)
1132
+
1133
+ assert_requested(elastic_request)
1121
1134
  end
1122
1135
 
1123
- def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
1136
+ def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
1124
1137
  cwd = File.dirname(__FILE__)
1125
1138
  template_file = File.join(cwd, 'test_template.json')
1126
1139
 
@@ -1134,9 +1147,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1134
1147
  template_name logstash
1135
1148
  template_file #{template_file}
1136
1149
  index_date_pattern now/w{xxxx.ww}
1137
- deflector_alias myapp_deflector
1138
1150
  index_name logstash
1139
1151
  enable_ilm true
1152
+ ilm_policy_overwrite true
1153
+ ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
1140
1154
  }
1141
1155
 
1142
1156
  # connection start
@@ -1152,38 +1166,63 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1152
1166
  with(basic_auth: ['john', 'doe']).
1153
1167
  to_return(:status => 200, :body => "", :headers => {})
1154
1168
  # check if alias exists
1155
- stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
1169
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1156
1170
  with(basic_auth: ['john', 'doe']).
1157
1171
  to_return(:status => 404, :body => "", :headers => {})
1158
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
1172
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1159
1173
  with(basic_auth: ['john', 'doe']).
1160
1174
  to_return(status: 404, body: "", headers: {})
1161
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
1175
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1162
1176
  with(basic_auth: ['john', 'doe'],
1163
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
1177
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1164
1178
  to_return(status: 200, body: "", headers: {})
1165
1179
  # put the alias for the index
1166
1180
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1167
1181
  with(basic_auth: ['john', 'doe']).
1168
1182
  to_return(:status => 200, :body => "", :headers => {})
1169
- stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
1183
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1170
1184
  with(basic_auth: ['john', 'doe'],
1171
- :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
1185
+ :body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1172
1186
  to_return(:status => 200, :body => "", :headers => {})
1173
1187
  stub_request(:get, "https://logs.google.com:777/es//_xpack").
1174
1188
  with(basic_auth: ['john', 'doe']).
1175
1189
  to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1176
1190
  stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1177
1191
  with(basic_auth: ['john', 'doe']).
1178
- to_return(:status => 404, :body => "", :headers => {})
1192
+ to_return(:status => 200, :body => "", :headers => {})
1179
1193
  stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1180
1194
  with(basic_auth: ['john', 'doe'],
1181
- :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1195
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"60gb\",\"max_age\":\"45d\"}}}}}}").
1182
1196
  to_return(:status => 200, :body => "", :headers => {})
1183
1197
 
1184
1198
  driver(config)
1185
1199
 
1186
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
1200
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1201
+ end
1202
+
1203
+ def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
1204
+ cwd = File.dirname(__FILE__)
1205
+ template_file = File.join(cwd, 'test_template.json')
1206
+
1207
+ config = %{
1208
+ host logs.google.com
1209
+ port 777
1210
+ scheme https
1211
+ path /es/
1212
+ user john
1213
+ password doe
1214
+ template_name logstash
1215
+ template_file #{template_file}
1216
+ index_date_pattern now/w{xxxx.ww}
1217
+ deflector_alias myapp_deflector
1218
+ index_name logstash
1219
+ enable_ilm true
1220
+ }
1221
+
1222
+ # Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
1223
+ assert_raise(Fluent::ConfigError) do
1224
+ driver(config)
1225
+ end
1187
1226
  end
1188
1227
 
1189
1228
  def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
@@ -1607,6 +1646,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1607
1646
 
1608
1647
  assert_requested(elastic_request)
1609
1648
  end
1649
+
1650
+ def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
1651
+ cwd = File.dirname(__FILE__)
1652
+ template_file = File.join(cwd, 'test_template.json')
1653
+
1654
+ config = Fluent::Config::Element.new(
1655
+ 'ROOT', '', {
1656
+ '@type' => 'elasticsearch',
1657
+ 'host' => 'logs.google.com',
1658
+ 'port' => 777,
1659
+ 'scheme' => "https",
1660
+ 'path' => "/es/",
1661
+ 'user' => 'john',
1662
+ 'password' => 'doe',
1663
+ 'template_name' => 'logstash',
1664
+ 'template_file' => "#{template_file}",
1665
+ 'index_date_pattern' => 'now/w{xxxx.ww}',
1666
+ 'index_name' => "${taskDef}-%Y.%m",
1667
+ 'enable_ilm' => true,
1668
+ }, [
1669
+ Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
1670
+ 'chunk_keys' => ['tag', 'time', 'taskDef'],
1671
+ 'timekey' => 3600,
1672
+ }, [])
1673
+ ]
1674
+ )
1675
+
1676
+ task_def_value = "task_definition"
1677
+ date_str = Time.now.strftime("%Y.%m")
1678
+ # connection start
1679
+ stub_request(:head, "https://logs.google.com:777/es//").
1680
+ with(basic_auth: ['john', 'doe']).
1681
+ to_return(:status => 200, :body => "", :headers => {})
1682
+ # check if template exists
1683
+ stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1684
+ with(basic_auth: ['john', 'doe']).
1685
+ to_return(:status => 404, :body => "", :headers => {})
1686
+ # creation
1687
+ stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1688
+ with(basic_auth: ['john', 'doe'],
1689
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
1690
+ to_return(:status => 200, :body => "", :headers => {})
1691
+ # check if alias exists
1692
+ stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
1693
+ with(basic_auth: ['john', 'doe']).
1694
+ to_return(:status => 404, :body => "", :headers => {})
1695
+ # put the alias for the index
1696
+ stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1697
+ with(basic_auth: ['john', 'doe']).
1698
+ to_return(:status => 200, :body => "", :headers => {})
1699
+ stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
1700
+ with(basic_auth: ['john', 'doe'],
1701
+ :body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
1702
+ to_return(:status => 200, :body => "", :headers => {})
1703
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1704
+ with(basic_auth: ['john', 'doe']).
1705
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1706
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1707
+ with(basic_auth: ['john', 'doe']).
1708
+ to_return(:status => 404, :body => "", :headers => {})
1709
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1710
+ with(basic_auth: ['john', 'doe'],
1711
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1712
+ to_return(:status => 200, :body => "", :headers => {})
1713
+
1714
+ driver(config)
1715
+
1716
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1717
+ driver.run(default_tag: 'test') do
1718
+ driver.feed(sample_record.merge("taskDef" => task_def_value))
1719
+ end
1720
+ assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
1721
+
1722
+ assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
1723
+
1724
+ assert_requested(elastic_request)
1725
+ end
1610
1726
  end
1611
1727
 
1612
1728
  def test_custom_template_create
@@ -1845,6 +1961,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1845
1961
  application_name myapp
1846
1962
  }
1847
1963
 
1964
+ timestr = Time.now.strftime("%Y.%m.%d")
1848
1965
  # connection start
1849
1966
  stub_request(:head, "https://logs.google.com:777/es//").
1850
1967
  with(basic_auth: ['john', 'doe']).
@@ -1858,17 +1975,17 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1858
1975
  with(basic_auth: ['john', 'doe']).
1859
1976
  to_return(:status => 200, :body => "", :headers => {})
1860
1977
  # creation of index which can rollover
1861
- stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1978
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E").
1862
1979
  with(basic_auth: ['john', 'doe']).
1863
1980
  to_return(:status => 200, :body => "", :headers => {})
1864
1981
  # check if alias exists
1865
- timestr = Time.now.strftime("%Y.%m.%d")
1866
1982
  stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
1867
1983
  with(basic_auth: ['john', 'doe']).
1868
1984
  to_return(:status => 404, :body => "", :headers => {})
1869
1985
  # put the alias for the index
1870
- stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
1871
- with(basic_auth: ['john', 'doe']).
1986
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
1987
+ with(basic_auth: ['john', 'doe'],
1988
+ body: "{\"aliases\":{\"mylogs-#{timestr}\":{\"is_write_index\":true}}}").
1872
1989
  to_return(:status => 200, :body => "", :headers => {})
1873
1990
 
1874
1991
  driver(config)
@@ -2051,52 +2168,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2051
2168
  enable_ilm true
2052
2169
  }
2053
2170
 
2054
- # connection start
2055
- stub_request(:head, "https://logs.google.com:777/es//").
2056
- with(basic_auth: ['john', 'doe']).
2057
- to_return(:status => 200, :body => "", :headers => {})
2058
- # check if template exists
2059
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2060
- with(basic_auth: ['john', 'doe']).
2061
- to_return(:status => 404, :body => "", :headers => {})
2062
- # creation
2063
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2064
- with(basic_auth: ['john', 'doe']).
2065
- to_return(:status => 200, :body => "", :headers => {})
2066
- # creation of index which can rollover
2067
- stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
2068
- with(basic_auth: ['john', 'doe']).
2069
- to_return(:status => 200, :body => "", :headers => {})
2070
- # check if alias exists
2071
- stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
2072
- with(basic_auth: ['john', 'doe']).
2073
- to_return(:status => 404, :body => "", :headers => {})
2074
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
2075
- with(basic_auth: ['john', 'doe']).
2076
- to_return(status: 404, body: "", headers: {})
2077
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
2078
- with(basic_auth: ['john', 'doe'],
2079
- body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
2080
- to_return(status: 200, body: "", headers: {})
2081
- # put the alias for the index
2082
- stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
2083
- with(basic_auth: ['john', 'doe'],
2084
- :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
2085
- to_return(:status => 200, :body => "", :headers => {})
2086
- stub_request(:get, "https://logs.google.com:777/es//_xpack").
2087
- with(basic_auth: ['john', 'doe']).
2088
- to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
2089
- stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
2090
- with(basic_auth: ['john', 'doe']).
2091
- to_return(:status => 404, :body => "", :headers => {})
2092
- stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
2093
- with(basic_auth: ['john', 'doe'],
2094
- :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
2095
- to_return(:status => 200, :body => "", :headers => {})
2096
-
2097
- driver(config)
2098
-
2099
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
2171
+ # Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
2172
+ assert_raise(Fluent::ConfigError) do
2173
+ driver(config)
2174
+ end
2100
2175
  end
2101
2176
 
2102
2177
  def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
@@ -3147,6 +3222,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
3147
3222
 
3148
3223
  data("border" => {"es_version" => 6, "_type" => "mytype"},
3149
3224
  "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
3225
+ "to be nil" => {"es_version" => 8, "_type" => nil},
3150
3226
  )
3151
3227
  def test_writes_to_speficied_type(data)
3152
3228
  driver('', data["es_version"]).configure("type_name mytype\n")
@@ -3159,6 +3235,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
3159
3235
 
3160
3236
  data("border" => {"es_version" => 6, "_type" => "mytype.test"},
3161
3237
  "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
3238
+ "to be nil" => {"es_version" => 8, "_type" => nil},
3162
3239
  )
3163
3240
  def test_writes_to_speficied_type_with_placeholders(data)
3164
3241
  driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
@@ -3169,6 +3246,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
3169
3246
  assert_equal(data['_type'], index_cmds.first['index']['_type'])
3170
3247
  end
3171
3248
 
3249
+ data("border" => {"es_version" => 6, "_type" => "mytype.test"},
3250
+ "fixed_behavior"=> {"es_version" => 7, "_type" => nil},
3251
+ "to be nil" => {"es_version" => 8, "_type" => nil},
3252
+ )
3253
+ def test_writes_to_speficied_type_with_suppress_type_name(data)
3254
+ driver('', data["es_version"])
3255
+ .configure("type_name mytype.${tag}\nsuppress_type_name true")
3256
+ stub_elastic
3257
+ driver.run(default_tag: 'test') do
3258
+ driver.feed(sample_record)
3259
+ end
3260
+ assert_equal(data['_type'], index_cmds.first['index']['_type'])
3261
+ end
3262
+
3172
3263
  data("old" => {"es_version" => 2, "_type" => "local-override"},
3173
3264
  "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
3174
3265
  "border" => {"es_version" => 6, "_type" => "fluentd"},
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.9
4
+ version: 4.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
8
8
  - pitr
9
9
  - Hiroshi Hatake
10
- autorequire:
10
+ autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-05-27 00:00:00.000000000 Z
13
+ date: 2020-08-19 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -135,6 +135,7 @@ extra_rdoc_files: []
135
135
  files:
136
136
  - ".coveralls.yml"
137
137
  - ".editorconfig"
138
+ - ".github/workflows/issue-auto-closer.yml"
138
139
  - ".github/workflows/linux.yml"
139
140
  - ".github/workflows/macos.yml"
140
141
  - ".github/workflows/windows.yml"
@@ -146,6 +147,7 @@ files:
146
147
  - ISSUE_TEMPLATE.md
147
148
  - LICENSE.txt
148
149
  - PULL_REQUEST_TEMPLATE.md
150
+ - README.ElasticsearchGenID.md
149
151
  - README.ElasticsearchInput.md
150
152
  - README.md
151
153
  - Rakefile
@@ -158,6 +160,7 @@ files:
158
160
  - lib/fluent/plugin/elasticsearch_constants.rb
159
161
  - lib/fluent/plugin/elasticsearch_error.rb
160
162
  - lib/fluent/plugin/elasticsearch_error_handler.rb
163
+ - lib/fluent/plugin/elasticsearch_fallback_selector.rb
161
164
  - lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb
162
165
  - lib/fluent/plugin/elasticsearch_index_template.rb
163
166
  - lib/fluent/plugin/elasticsearch_simple_sniffer.rb
@@ -170,6 +173,7 @@ files:
170
173
  - test/helper.rb
171
174
  - test/plugin/test_alias_template.json
172
175
  - test/plugin/test_elasticsearch_error_handler.rb
176
+ - test/plugin/test_elasticsearch_fallback_selector.rb
173
177
  - test/plugin/test_elasticsearch_index_lifecycle_management.rb
174
178
  - test/plugin/test_elasticsearch_tls.rb
175
179
  - test/plugin/test_filter_elasticsearch_genid.rb
@@ -184,7 +188,7 @@ licenses:
184
188
  - Apache-2.0
185
189
  metadata:
186
190
  changelog_uri: https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md
187
- post_install_message:
191
+ post_install_message:
188
192
  rdoc_options: []
189
193
  require_paths:
190
194
  - lib
@@ -200,13 +204,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
200
204
  version: '0'
201
205
  requirements: []
202
206
  rubygems_version: 3.1.2
203
- signing_key:
207
+ signing_key:
204
208
  specification_version: 4
205
209
  summary: Elasticsearch output plugin for Fluent event collector
206
210
  test_files:
207
211
  - test/helper.rb
208
212
  - test/plugin/test_alias_template.json
209
213
  - test/plugin/test_elasticsearch_error_handler.rb
214
+ - test/plugin/test_elasticsearch_fallback_selector.rb
210
215
  - test/plugin/test_elasticsearch_index_lifecycle_management.rb
211
216
  - test/plugin/test_elasticsearch_tls.rb
212
217
  - test/plugin/test_filter_elasticsearch_genid.rb