fluent-plugin-elasticsearch 4.0.5 → 4.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- require 'helper'
1
+ require_relative '../helper'
2
2
  require 'fluent/plugin/out_elasticsearch'
3
3
  require 'fluent/plugin/elasticsearch_error_handler'
4
4
  require 'json'
@@ -1,4 +1,4 @@
1
- require 'helper'
1
+ require_relative '../helper'
2
2
  require 'elasticsearch'
3
3
  require 'fluent/plugin/elasticsearch_index_lifecycle_management'
4
4
 
@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
62
62
  d = driver('')
63
63
  ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
64
64
  if @use_tls_minmax_version
65
- assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION,
66
- min_version: OpenSSL::SSL::TLS1_VERSION}, ssl_version_options)
65
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
66
+ min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
67
67
  else
68
68
  assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
69
69
  end
@@ -1,4 +1,4 @@
1
- require 'helper'
1
+ require_relative '../helper'
2
2
  require 'date'
3
3
  require 'fluent/test/helpers'
4
4
  require 'json'
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
18
18
  Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
19
19
  end
20
20
 
21
+ test "invalid configuration" do
22
+ assert_raise(Fluent::ConfigError) do
23
+ create_driver("use_record_as_seed true")
24
+ end
25
+ end
26
+
21
27
  def sample_record
22
28
  {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
23
29
  end
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
41
47
  assert_equal(Base64.strict_encode64(SecureRandom.uuid),
42
48
  d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
43
49
  end
50
+
51
+ class UseRecordAsSeedTest < self
52
+ data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
53
+ "sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
54
+ "sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
55
+ "sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
56
+ )
57
+ def test_simple(data)
58
+ hash_type, expected = data
59
+ d = create_driver(%[
60
+ use_record_as_seed true
61
+ record_keys age,parent_id,routing_id,custom_key
62
+ hash_type #{hash_type}
63
+ ])
64
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
65
+ d.run(default_tag: 'test') do
66
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
67
+ end
68
+ assert_equal(expected,
69
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
70
+ end
71
+
72
+ data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
73
+ "sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
74
+ "sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
75
+ "sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
76
+ def test_record_with_tag(data)
77
+ hash_type, expected = data
78
+ d = create_driver(%[
79
+ use_record_as_seed true
80
+ record_keys age,parent_id,routing_id,custom_key
81
+ hash_type #{hash_type}
82
+ include_tag_in_seed true
83
+ ])
84
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
85
+ d.run(default_tag: 'test.fluentd') do
86
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
87
+ end
88
+ assert_equal(expected,
89
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
90
+ end
91
+
92
+ data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
93
+ "sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
94
+ "sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
95
+ "sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
96
+ def test_record_with_time(data)
97
+ hash_type, expected = data
98
+ d = create_driver(%[
99
+ use_record_as_seed true
100
+ record_keys age,parent_id,routing_id,custom_key
101
+ hash_type #{hash_type}
102
+ include_time_in_seed true
103
+ ])
104
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
105
+ d.run(default_tag: 'test.fluentd') do
106
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
107
+ end
108
+ assert_equal(expected,
109
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
110
+ end
111
+
112
+ data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
113
+ "sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
114
+ "sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
115
+ "sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
116
+ def test_record_with_tag_and_time
117
+ hash_type, expected = data
118
+ d = create_driver(%[
119
+ use_record_as_seed true
120
+ record_keys age,parent_id,routing_id,custom_key
121
+ hash_type #{hash_type}
122
+ include_tag_in_seed true
123
+ include_time_in_seed true
124
+ ])
125
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
126
+ d.run(default_tag: 'test.fluentd') do
127
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
128
+ end
129
+ assert_equal(expected,
130
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
131
+ end
132
+ end
133
+
134
+ class UseEntireRecordAsSeedTest < self
135
+ data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
136
+ "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
137
+ "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
138
+ "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
139
+ def test_record
140
+ hash_type, expected = data
141
+ d = create_driver(%[
142
+ use_record_as_seed true
143
+ use_entire_record true
144
+ hash_type #{hash_type}
145
+ ])
146
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
147
+ d.run(default_tag: 'test.fluentd') do
148
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
149
+ end
150
+ assert_equal(expected,
151
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
152
+ end
153
+
154
+ data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
155
+ "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
156
+ "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
157
+ "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
158
+ def test_record_with_tag
159
+ hash_type, expected = data
160
+ d = create_driver(%[
161
+ use_record_as_seed true
162
+ use_entire_record true
163
+ hash_type #{hash_type}
164
+ include_tag_in_seed true
165
+ ])
166
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
167
+ d.run(default_tag: 'test.fluentd') do
168
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
169
+ end
170
+ assert_equal(expected,
171
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
172
+ end
173
+
174
+ data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
175
+ "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
176
+ "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
177
+ "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
178
+ def test_record_with_time
179
+ hash_type, expected = data
180
+ d = create_driver(%[
181
+ use_record_as_seed true
182
+ use_entire_record true
183
+ hash_type #{hash_type}
184
+ include_time_in_seed true
185
+ ])
186
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
187
+ d.run(default_tag: 'test.fluentd') do
188
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
189
+ end
190
+ assert_equal(expected,
191
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
192
+ end
193
+
194
+ data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
195
+ "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
196
+ "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
197
+ "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
198
+ def test_record_with_tag_and_time
199
+ hash_type, expected = data
200
+ d = create_driver(%[
201
+ use_record_as_seed true
202
+ use_entire_record true
203
+ hash_type #{hash_type}
204
+ include_tag_in_seed true
205
+ include_time_in_seed true
206
+ ])
207
+ time = event_time("2017-10-15 15:00:23.34567890 UTC")
208
+ d.run(default_tag: 'test.fluentd') do
209
+ d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
210
+ end
211
+ assert_equal(expected,
212
+ d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
213
+ end
214
+ end
44
215
  end
@@ -4,6 +4,7 @@ require 'fluent/test/helpers'
4
4
  require 'json'
5
5
  require 'fluent/test/driver/input'
6
6
  require 'flexmock/test_unit'
7
+ require 'fluent/plugin/in_elasticsearch'
7
8
 
8
9
  class ElasticsearchInputTest < Test::Unit::TestCase
9
10
  include FlexMock::TestCase
@@ -16,7 +17,6 @@ class ElasticsearchInputTest < Test::Unit::TestCase
16
17
 
17
18
  def setup
18
19
  Fluent::Test.setup
19
- require 'fluent/plugin/in_elasticsearch'
20
20
  @driver = nil
21
21
  log = Fluent::Engine.log
22
22
  log.out.logs.slice!(0, log.out.logs.length)
@@ -1,11 +1,12 @@
1
- require 'helper'
1
+ require_relative '../helper'
2
2
  require 'date'
3
3
  require 'fluent/test/helpers'
4
4
  require 'json'
5
5
  require 'fluent/test/driver/output'
6
6
  require 'flexmock/test_unit'
7
+ require 'fluent/plugin/out_elasticsearch'
7
8
 
8
- class ElasticsearchOutput < Test::Unit::TestCase
9
+ class ElasticsearchOutputTest < Test::Unit::TestCase
9
10
  include FlexMock::TestCase
10
11
  include Fluent::Test::Helpers
11
12
 
@@ -13,7 +14,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
13
14
 
14
15
  def setup
15
16
  Fluent::Test.setup
16
- require 'fluent/plugin/out_elasticsearch'
17
17
  @driver = nil
18
18
  log = Fluent::Engine.log
19
19
  log.out.logs.slice!(0, log.out.logs.length)
@@ -230,7 +230,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
230
230
  assert_nil instance.ssl_max_version
231
231
  assert_nil instance.ssl_min_version
232
232
  if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
233
- assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
233
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
234
234
  instance.ssl_version_options)
235
235
  else
236
236
  assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
@@ -251,6 +251,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
251
251
  assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
252
252
  assert_false instance.compression
253
253
  assert_equal :no_compression, instance.compression_level
254
+ assert_true instance.http_backend_excon_nonblock
254
255
  end
255
256
 
256
257
  test 'configure compression' do
@@ -283,7 +284,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
283
284
  }
284
285
  instance = driver(config).instance
285
286
 
286
- assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
287
+ assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
288
+
289
+ stub_request(:post, "http://localhost:9200/_bulk").
290
+ to_return(status: 200, body: "", headers: {})
291
+ driver.run(default_tag: 'test') do
292
+ driver.feed(sample_record)
293
+ end
294
+ compressable = instance.compressable_connection
295
+
296
+ assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
287
297
  end
288
298
 
289
299
  test 'check compression option is passed to transport' do
@@ -294,7 +304,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
294
304
  }
295
305
  instance = driver(config).instance
296
306
 
297
- assert_equal true, instance.client.transport.options[:compression]
307
+ assert_equal false, instance.client.transport.options[:compression]
308
+
309
+ stub_request(:post, "http://localhost:9200/_bulk").
310
+ to_return(status: 200, body: "", headers: {})
311
+ driver.run(default_tag: 'test') do
312
+ driver.feed(sample_record)
313
+ end
314
+ compressable = instance.compressable_connection
315
+
316
+ assert_equal true, instance.client(nil, compressable).transport.options[:compression]
298
317
  end
299
318
 
300
319
  test 'configure Content-Type' do
@@ -332,6 +351,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
332
351
  }
333
352
  end
334
353
 
354
+ sub_test_case 'Check TLS handshake stuck warning log' do
355
+ test 'warning TLS log' do
356
+ config = %{
357
+ scheme https
358
+ http_backend_excon_nonblock false
359
+ ssl_version TLSv1_2
360
+ @log_level info
361
+ }
362
+ driver(config)
363
+ logs = driver.logs
364
+ assert_logs_include(logs, /TLS handshake will be stucked with block connection.\n Consider to set `http_backend_excon_nonblock` as true\n/)
365
+ end
366
+ end
367
+
335
368
  sub_test_case 'ILM default config' do
336
369
  setup do
337
370
  begin
@@ -535,6 +568,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
535
568
  "template_name_placeholder" => ["template_name", "logstash-${mykey}"],
536
569
  "customize_template" => ["customize_template", '{"<<TAG>>":"${mykey}"}'],
537
570
  "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${mykey}"],
571
+ "logstash_dateformat_placeholder" => ["logstash_dateformat", "${mykey}"],
538
572
  "deflector_alias_placeholder" => ["deflector_alias", "fluentd-${mykey}"],
539
573
  "application_name_placeholder" => ["application_name", "fluentd-${mykey}"],
540
574
  )
@@ -560,6 +594,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
560
594
  "template_name_placeholder" => ["template_name", "logstash-${tag}-%Y%m%d"],
561
595
  "customize_template" => ["customize_template", '{"<<TAG>>":"${es_index}"}'],
562
596
  "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${es_index}-%Y%m%d"],
597
+ "logstash_dataformat_placeholder" => ["logstash_dateformat", "${es_index}"],
563
598
  "deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
564
599
  "application_name_placeholder" => ["application_name", "fluentd-${tag}-${es_index}-%Y%m%d"],
565
600
  )
@@ -710,6 +745,60 @@ class ElasticsearchOutput < Test::Unit::TestCase
710
745
  end
711
746
  end
712
747
 
748
+ class GetElasticsearchVersionWithFallbackTest < self
749
+ def create_driver(conf='', client_version="\"5.0\"")
750
+ # For request stub to detect compatibility.
751
+ @client_version ||= client_version
752
+ # Ensure original implementation existence.
753
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
754
+ def detect_es_major_version
755
+ @_es_info ||= client.info
756
+ @_es_info["version"]["number"].to_i
757
+ end
758
+ CODE
759
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
760
+ def client_library_version
761
+ #{@client_version}
762
+ end
763
+ CODE
764
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
765
+ end
766
+
767
+ data("Elasticsearch 5" => ["5.0", 5],
768
+ "Elasticsearch 6" => ["6.0", 6],
769
+ "Elasticsearch 7" => ["7.0", 7],
770
+ "Elasticsearch 8" => ["8.0", 8])
771
+ def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
772
+ client_version, es_major_version = data
773
+ config = %{
774
+ host logs.google.com
775
+ port 778
776
+ scheme https
777
+ path /es/
778
+ user john
779
+ password doe
780
+ verify_es_version_at_startup true
781
+ max_retry_get_es_version 2
782
+ fail_on_detecting_es_version_retry_exceed false
783
+ default_elasticsearch_version #{es_major_version}
784
+ @log_level info
785
+ }
786
+
787
+ connection_resets = 0
788
+ stub_request(:get, "https://logs.google.com:778/es//").
789
+ with(basic_auth: ['john', 'doe']) do |req|
790
+ connection_resets += 1
791
+ raise Faraday::ConnectionFailed, "Test message"
792
+ end
793
+
794
+ d = create_driver(config, client_version)
795
+
796
+ assert_equal es_major_version, d.instance.default_elasticsearch_version
797
+ assert_equal 3, connection_resets
798
+ assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
799
+ end
800
+ end
801
+
713
802
  def test_template_already_present
714
803
  config = %{
715
804
  host logs.google.com
@@ -1228,6 +1317,224 @@ class ElasticsearchOutput < Test::Unit::TestCase
1228
1317
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1229
1318
  end
1230
1319
 
1320
+ def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
1321
+ cwd = File.dirname(__FILE__)
1322
+ template_file = File.join(cwd, 'test_template.json')
1323
+
1324
+ config = %{
1325
+ host logs.google.com
1326
+ port 777
1327
+ scheme https
1328
+ path /es/
1329
+ user john
1330
+ password doe
1331
+ template_name logstash
1332
+ template_file #{template_file}
1333
+ index_date_pattern now/w{xxxx.ww}
1334
+ ilm_policy_id fluentd-policy
1335
+ enable_ilm true
1336
+ index_name logstash
1337
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1338
+ }
1339
+
1340
+ # connection start
1341
+ stub_request(:head, "https://logs.google.com:777/es//").
1342
+ with(basic_auth: ['john', 'doe']).
1343
+ to_return(:status => 200, :body => "", :headers => {})
1344
+ # check if template exists
1345
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1346
+ with(basic_auth: ['john', 'doe']).
1347
+ to_return(:status => 404, :body => "", :headers => {})
1348
+ # creation
1349
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1350
+ with(basic_auth: ['john', 'doe']).
1351
+ to_return(:status => 200, :body => "", :headers => {})
1352
+ # check if alias exists
1353
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1354
+ with(basic_auth: ['john', 'doe']).
1355
+ to_return(:status => 404, :body => "", :headers => {})
1356
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1357
+ with(basic_auth: ['john', 'doe']).
1358
+ to_return(status: 404, body: "", headers: {})
1359
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1360
+ with(basic_auth: ['john', 'doe'],
1361
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1362
+ to_return(status: 200, body: "", headers: {})
1363
+ # put the alias for the index
1364
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1365
+ with(basic_auth: ['john', 'doe']).
1366
+ to_return(:status => 200, :body => "", :headers => {})
1367
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1368
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1369
+ to_return(status: 200, body: "", headers: {})
1370
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1371
+ with(basic_auth: ['john', 'doe']).
1372
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1373
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1374
+ with(basic_auth: ['john', 'doe']).
1375
+ to_return(:status => 404, :body => "", :headers => {})
1376
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1377
+ with(basic_auth: ['john', 'doe'],
1378
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1379
+ to_return(:status => 200, :body => "", :headers => {})
1380
+
1381
+ driver(config)
1382
+
1383
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1384
+ driver.run(default_tag: 'test') do
1385
+ driver.feed(sample_record)
1386
+ end
1387
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1388
+
1389
+ assert_requested(elastic_request)
1390
+ end
1391
+
1392
+ class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
1393
+ def test_tag_placeholder
1394
+ cwd = File.dirname(__FILE__)
1395
+ template_file = File.join(cwd, 'test_template.json')
1396
+
1397
+ config = %{
1398
+ host logs.google.com
1399
+ port 777
1400
+ scheme https
1401
+ path /es/
1402
+ user john
1403
+ password doe
1404
+ template_name logstash
1405
+ template_file #{template_file}
1406
+ index_date_pattern now/w{xxxx.ww}
1407
+ ilm_policy_id ${tag}
1408
+ enable_ilm true
1409
+ index_name logstash
1410
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1411
+ }
1412
+
1413
+ # connection start
1414
+ stub_request(:head, "https://logs.google.com:777/es//").
1415
+ with(basic_auth: ['john', 'doe']).
1416
+ to_return(:status => 200, :body => "", :headers => {})
1417
+ # check if template exists
1418
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1419
+ with(basic_auth: ['john', 'doe']).
1420
+ to_return(:status => 404, :body => "", :headers => {})
1421
+ # creation
1422
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1423
+ with(basic_auth: ['john', 'doe']).
1424
+ to_return(:status => 200, :body => "", :headers => {})
1425
+ # check if alias exists
1426
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1427
+ with(basic_auth: ['john', 'doe']).
1428
+ to_return(:status => 404, :body => "", :headers => {})
1429
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1430
+ with(basic_auth: ['john', 'doe']).
1431
+ to_return(status: 404, body: "", headers: {})
1432
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1433
+ with(basic_auth: ['john', 'doe'],
1434
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1435
+ to_return(status: 200, body: "", headers: {})
1436
+ # put the alias for the index
1437
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1438
+ with(basic_auth: ['john', 'doe']).
1439
+ to_return(:status => 200, :body => "", :headers => {})
1440
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1441
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1442
+ to_return(status: 200, body: "", headers: {})
1443
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1444
+ with(basic_auth: ['john', 'doe']).
1445
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1446
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1447
+ with(basic_auth: ['john', 'doe']).
1448
+ to_return(:status => 404, :body => "", :headers => {})
1449
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1450
+ with(basic_auth: ['john', 'doe'],
1451
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1452
+ to_return(:status => 200, :body => "", :headers => {})
1453
+
1454
+ driver(config)
1455
+
1456
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1457
+ driver.run(default_tag: 'fluentd-policy') do
1458
+ driver.feed(sample_record)
1459
+ end
1460
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1461
+
1462
+ assert_requested(elastic_request)
1463
+ end
1464
+
1465
+ def test_tag_placeholder_with_multiple_policies
1466
+ cwd = File.dirname(__FILE__)
1467
+ template_file = File.join(cwd, 'test_template.json')
1468
+
1469
+ config = %{
1470
+ host logs.google.com
1471
+ port 777
1472
+ scheme https
1473
+ path /es/
1474
+ user john
1475
+ password doe
1476
+ template_name logstash
1477
+ template_file #{template_file}
1478
+ index_date_pattern now/w{xxxx.ww}
1479
+ ilm_policy_id ${tag}
1480
+ enable_ilm true
1481
+ index_name logstash
1482
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
1483
+ }
1484
+
1485
+ # connection start
1486
+ stub_request(:head, "https://logs.google.com:777/es//").
1487
+ with(basic_auth: ['john', 'doe']).
1488
+ to_return(:status => 200, :body => "", :headers => {})
1489
+ # check if template exists
1490
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1491
+ with(basic_auth: ['john', 'doe']).
1492
+ to_return(:status => 404, :body => "", :headers => {})
1493
+ # creation
1494
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1495
+ with(basic_auth: ['john', 'doe']).
1496
+ to_return(:status => 200, :body => "", :headers => {})
1497
+ # check if alias exists
1498
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1499
+ with(basic_auth: ['john', 'doe']).
1500
+ to_return(:status => 404, :body => "", :headers => {})
1501
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1502
+ with(basic_auth: ['john', 'doe']).
1503
+ to_return(status: 404, body: "", headers: {})
1504
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1505
+ with(basic_auth: ['john', 'doe'],
1506
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1507
+ to_return(status: 200, body: "", headers: {})
1508
+ # put the alias for the index
1509
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1510
+ with(basic_auth: ['john', 'doe']).
1511
+ to_return(:status => 200, :body => "", :headers => {})
1512
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1513
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1514
+ to_return(status: 200, body: "", headers: {})
1515
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1516
+ with(basic_auth: ['john', 'doe']).
1517
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1518
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1519
+ with(basic_auth: ['john', 'doe']).
1520
+ to_return(:status => 404, :body => "", :headers => {})
1521
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1522
+ with(basic_auth: ['john', 'doe'],
1523
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
1524
+ to_return(:status => 200, :body => "", :headers => {})
1525
+
1526
+ driver(config)
1527
+
1528
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1529
+ driver.run(default_tag: 'fluentd-policy2') do
1530
+ driver.feed(sample_record)
1531
+ end
1532
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1533
+
1534
+ assert_requested(elastic_request)
1535
+ end
1536
+ end
1537
+
1231
1538
  def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
1232
1539
  cwd = File.dirname(__FILE__)
1233
1540
  template_file = File.join(cwd, 'test_template.json')
@@ -2840,6 +3147,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
2840
3147
 
2841
3148
  data("border" => {"es_version" => 6, "_type" => "mytype"},
2842
3149
  "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
3150
+ "to be nil" => {"es_version" => 8, "_type" => nil},
2843
3151
  )
2844
3152
  def test_writes_to_speficied_type(data)
2845
3153
  driver('', data["es_version"]).configure("type_name mytype\n")
@@ -2852,6 +3160,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
2852
3160
 
2853
3161
  data("border" => {"es_version" => 6, "_type" => "mytype.test"},
2854
3162
  "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
3163
+ "to be nil" => {"es_version" => 8, "_type" => nil},
2855
3164
  )
2856
3165
  def test_writes_to_speficied_type_with_placeholders(data)
2857
3166
  driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
@@ -2862,6 +3171,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
2862
3171
  assert_equal(data['_type'], index_cmds.first['index']['_type'])
2863
3172
  end
2864
3173
 
3174
+ data("border" => {"es_version" => 6, "_type" => "mytype.test"},
3175
+ "fixed_behavior"=> {"es_version" => 7, "_type" => nil},
3176
+ "to be nil" => {"es_version" => 8, "_type" => nil},
3177
+ )
3178
+ def test_writes_to_speficied_type_with_suppress_type_name(data)
3179
+ driver('', data["es_version"])
3180
+ .configure("type_name mytype.${tag}\nsuppress_type_name true")
3181
+ stub_elastic
3182
+ driver.run(default_tag: 'test') do
3183
+ driver.feed(sample_record)
3184
+ end
3185
+ assert_equal(data['_type'], index_cmds.first['index']['_type'])
3186
+ end
3187
+
2865
3188
  data("old" => {"es_version" => 2, "_type" => "local-override"},
2866
3189
  "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
2867
3190
  "border" => {"es_version" => 6, "_type" => "fluentd"},