fluent-plugin-elasticsearch 2.10.3 → 2.10.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (27) hide show
  1. checksums.yaml +4 -4
  2. data/.coveralls.yml +2 -2
  3. data/.editorconfig +9 -9
  4. data/.gitignore +18 -18
  5. data/.travis.yml +14 -14
  6. data/Gemfile +8 -8
  7. data/History.md +243 -239
  8. data/ISSUE_TEMPLATE.md +25 -25
  9. data/LICENSE.txt +201 -201
  10. data/PULL_REQUEST_TEMPLATE.md +10 -10
  11. data/README.md +807 -807
  12. data/Rakefile +11 -11
  13. data/appveyor.yml +35 -35
  14. data/fluent-plugin-elasticsearch.gemspec +31 -31
  15. data/lib/fluent/plugin/elasticsearch_constants.rb +13 -13
  16. data/lib/fluent/plugin/elasticsearch_error_handler.rb +89 -87
  17. data/lib/fluent/plugin/elasticsearch_index_template.rb +60 -60
  18. data/lib/fluent/plugin/filter_elasticsearch_genid.rb +25 -25
  19. data/lib/fluent/plugin/out_elasticsearch.rb +540 -538
  20. data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +294 -294
  21. data/test/helper.rb +24 -24
  22. data/test/plugin/test_elasticsearch_error_handler.rb +195 -195
  23. data/test/plugin/test_filter_elasticsearch_genid.rb +44 -44
  24. data/test/plugin/test_out_elasticsearch.rb +2055 -2046
  25. data/test/plugin/test_out_elasticsearch_dynamic.rb +992 -992
  26. data/test/plugin/test_template.json +23 -23
  27. metadata +3 -3
data/test/plugin/test_out_elasticsearch.rb CHANGED
@@ -1,2046 +1,2055 @@
1
- require 'helper'
2
- require 'date'
3
- require 'fluent/test/helpers'
4
- require 'json'
5
- require 'fluent/test/driver/output'
6
- require 'flexmock/test_unit'
7
-
8
- class ElasticsearchOutput < Test::Unit::TestCase
9
- include FlexMock::TestCase
10
- include Fluent::Test::Helpers
11
-
12
- attr_accessor :index_cmds, :index_command_counts
13
-
14
- def setup
15
- Fluent::Test.setup
16
- require 'fluent/plugin/out_elasticsearch'
17
- @driver = nil
18
- log = Fluent::Engine.log
19
- log.out.logs.slice!(0, log.out.logs.length)
20
- end
21
-
22
- def driver(conf='', es_version=5)
23
- # For request stub to detect compatibility.
24
- @es_version ||= es_version
25
- Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
26
- def detect_es_major_version
27
- #{@es_version}
28
- end
29
- CODE
30
- @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
31
- # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
32
- if !defined?(Fluent::Plugin::Output)
33
- def format(tag, time, record)
34
- [time, record].to_msgpack
35
- end
36
- end
37
- }.configure(conf)
38
- end
39
-
40
- def default_type_name
41
- Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
42
- end
43
-
44
- def sample_record
45
- {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
46
- end
47
-
48
- def nested_sample_record
49
- {'nested' =>
50
- {'age' => 26, 'parent_id' => 'parent', 'routing_id' => 'routing', 'request_id' => '42'}
51
- }
52
- end
53
-
54
- def stub_elastic_ping(url="http://localhost:9200")
55
- stub_request(:head, url).to_return(:status => 200, :body => "", :headers => {})
56
- end
57
-
58
- def stub_elastic(url="http://localhost:9200/_bulk")
59
- stub_request(:post, url).with do |req|
60
- @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
61
- end
62
- end
63
-
64
- def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
65
- stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
66
- end
67
-
68
- def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
69
- if @index_command_counts == nil
70
- @index_command_counts = {}
71
- @index_command_counts.default = 0
72
- end
73
-
74
- stub_request(:post, url).with do |req|
75
- index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
76
- @index_command_counts[url] += index_cmds.size
77
- end
78
- end
79
-
80
- def make_response_body(req, error_el = nil, error_status = nil, error = nil)
81
- req_index_cmds = req.body.split("\n").map { |r| JSON.parse(r) }
82
- items = []
83
- count = 0
84
- ids = 1
85
- op = nil
86
- index = nil
87
- type = nil
88
- id = nil
89
- req_index_cmds.each do |cmd|
90
- if count.even?
91
- op = cmd.keys[0]
92
- index = cmd[op]['_index']
93
- type = cmd[op]['_type']
94
- if cmd[op].has_key?('_id')
95
- id = cmd[op]['_id']
96
- else
97
- # Note: this appears to be an undocumented feature of Elasticsearch
98
- # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/docs-bulk.html
99
- # When you submit an "index" write_operation, with no "_id" field in the
100
- # metadata header, Elasticsearch will turn this into a "create"
101
- # operation in the response.
102
- if "index" == op
103
- op = "create"
104
- end
105
- id = ids
106
- ids += 1
107
- end
108
- else
109
- item = {
110
- op => {
111
- '_index' => index, '_type' => type, '_id' => id, '_version' => 1,
112
- '_shards' => { 'total' => 1, 'successful' => 1, 'failed' => 0 },
113
- 'status' => op == 'create' ? 201 : 200
114
- }
115
- }
116
- items.push(item)
117
- end
118
- count += 1
119
- end
120
- if !error_el.nil? && !error_status.nil? && !error.nil?
121
- op = items[error_el].keys[0]
122
- items[error_el][op].delete('_version')
123
- items[error_el][op].delete('_shards')
124
- items[error_el][op]['error'] = error
125
- items[error_el][op]['status'] = error_status
126
- errors = true
127
- else
128
- errors = false
129
- end
130
- @index_cmds = items
131
- body = { 'took' => 6, 'errors' => errors, 'items' => items }
132
- return body.to_json
133
- end
134
-
135
- def stub_elastic_bad_argument(url="http://localhost:9200/_bulk")
136
- error = {
137
- "type" => "mapper_parsing_exception",
138
- "reason" => "failed to parse [...]",
139
- "caused_by" => {
140
- "type" => "illegal_argument_exception",
141
- "reason" => "Invalid format: \"...\""
142
- }
143
- }
144
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 400, error), :headers => { 'Content-Type' => 'json' } } })
145
- end
146
-
147
- def stub_elastic_bulk_error(url="http://localhost:9200/_bulk")
148
- error = {
149
- "type" => "some-unrecognized-error",
150
- "reason" => "some message printed here ...",
151
- }
152
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
153
- end
154
-
155
- def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
156
- error = {
157
- "status" => 500,
158
- "type" => "es_rejected_execution_exception",
159
- "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
160
- }
161
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 429, error), :headers => { 'Content-Type' => 'json' } } })
162
- end
163
-
164
- def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
165
- error = {
166
- "status" => 500,
167
- "type" => "out_of_memory_error",
168
- "reason" => "Java heap space"
169
- }
170
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
171
- end
172
-
173
- def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
174
- error = {
175
- "category" => "some-other-type",
176
- "reason" => "some-other-reason"
177
- }
178
- stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
179
- end
180
-
181
- def assert_logs_include(logs, msg)
182
- matches = logs.grep /#{msg}/
183
- assert_equal(1, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
184
- end
185
-
186
- def test_configure
187
- config = %{
188
- host logs.google.com
189
- port 777
190
- scheme https
191
- path /es/
192
- user john
193
- password doe
194
- }
195
- instance = driver(config).instance
196
-
197
- assert_equal 'logs.google.com', instance.host
198
- assert_equal 777, instance.port
199
- assert_equal 'https', instance.scheme
200
- assert_equal '/es/', instance.path
201
- assert_equal 'john', instance.user
202
- assert_equal 'doe', instance.password
203
- assert_equal :TLSv1, instance.ssl_version
204
- assert_nil instance.client_key
205
- assert_nil instance.client_cert
206
- assert_nil instance.client_key_pass
207
- assert_false instance.with_transporter_log
208
- assert_equal :"application/json", instance.content_type
209
- assert_equal "fluentd", default_type_name
210
- end
211
-
212
- test 'configure Content-Type' do
213
- config = %{
214
- content_type application/x-ndjson
215
- }
216
- instance = driver(config).instance
217
- assert_equal :"application/x-ndjson", instance.content_type
218
- end
219
-
220
- test 'invalid Content-Type' do
221
- config = %{
222
- content_type nonexistent/invalid
223
- }
224
- assert_raise(Fluent::ConfigError) {
225
- instance = driver(config).instance
226
- }
227
- end
228
-
229
- test 'Detected Elasticsearch 7' do
230
- config = %{
231
- type_name changed
232
- }
233
- instance = driver(config, 7).instance
234
- assert_equal '_doc', instance.type_name
235
- end
236
-
237
- test 'lack of tag in chunk_keys' do
238
- assert_raise_message(/'tag' in chunk_keys is required./) do
239
- driver(Fluent::Config::Element.new(
240
- 'ROOT', '', {
241
- '@type' => 'elasticsearch',
242
- 'host' => 'log.google.com',
243
- 'port' => 777,
244
- 'scheme' => 'https',
245
- 'path' => '/es/',
246
- 'user' => 'john',
247
- 'pasword' => 'doe',
248
- }, [
249
- Fluent::Config::Element.new('buffer', 'mykey', {
250
- 'chunk_keys' => 'mykey'
251
- }, [])
252
- ]
253
- ))
254
- end
255
- end
256
-
257
- def test_template_already_present
258
- config = %{
259
- host logs.google.com
260
- port 777
261
- scheme https
262
- path /es/
263
- user john
264
- password doe
265
- template_name logstash
266
- template_file /abc123
267
- }
268
-
269
- # connection start
270
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
271
- to_return(:status => 200, :body => "", :headers => {})
272
- # check if template exists
273
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
274
- to_return(:status => 200, :body => "", :headers => {})
275
-
276
- driver(config)
277
-
278
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
279
- end
280
-
281
- def test_template_create
282
- cwd = File.dirname(__FILE__)
283
- template_file = File.join(cwd, 'test_template.json')
284
-
285
- config = %{
286
- host logs.google.com
287
- port 777
288
- scheme https
289
- path /es/
290
- user john
291
- password doe
292
- template_name logstash
293
- template_file #{template_file}
294
- }
295
-
296
- # connection start
297
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
298
- to_return(:status => 200, :body => "", :headers => {})
299
- # check if template exists
300
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
301
- to_return(:status => 404, :body => "", :headers => {})
302
- # creation
303
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
304
- to_return(:status => 200, :body => "", :headers => {})
305
-
306
- driver(config)
307
-
308
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
309
- end
310
-
311
- def test_template_overwrite
312
- cwd = File.dirname(__FILE__)
313
- template_file = File.join(cwd, 'test_template.json')
314
-
315
- config = %{
316
- host logs.google.com
317
- port 777
318
- scheme https
319
- path /es/
320
- user john
321
- password doe
322
- template_name logstash
323
- template_file #{template_file}
324
- template_overwrite true
325
- }
326
-
327
- # connection start
328
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
329
- to_return(:status => 200, :body => "", :headers => {})
330
- # check if template exists
331
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
332
- to_return(:status => 200, :body => "", :headers => {})
333
- # creation
334
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
335
- to_return(:status => 200, :body => "", :headers => {})
336
-
337
- driver(config)
338
-
339
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
340
- end
341
-
342
-
343
- def test_template_create_invalid_filename
344
- config = %{
345
- host logs.google.com
346
- port 777
347
- scheme https
348
- path /es/
349
- user john
350
- password doe
351
- template_name logstash
352
- template_file /abc123
353
- }
354
-
355
- # connection start
356
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
357
- to_return(:status => 200, :body => "", :headers => {})
358
- # check if template exists
359
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
360
- to_return(:status => 404, :body => "", :headers => {})
361
-
362
- assert_raise(RuntimeError) {
363
- driver(config)
364
- }
365
- end
366
-
367
- def test_template_retry_install
368
- cwd = File.dirname(__FILE__)
369
- template_file = File.join(cwd, 'test_template.json')
370
-
371
- config = %{
372
- host logs.google.com
373
- port 778
374
- scheme https
375
- path /es/
376
- user john
377
- password doe
378
- template_name logstash
379
- template_file #{template_file}
380
- max_retry_putting_template 3
381
- }
382
-
383
- connection_resets = 0
384
- # connection start
385
- stub_request(:head, "https://john:doe@logs.google.com:778/es//").with do |req|
386
- connection_resets += 1
387
- end
388
- # check if template exists
389
- stub_request(:get, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
390
- raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
391
- end
392
- # creation
393
- stub_request(:put, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
394
- raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
395
- end
396
-
397
- assert_raise(Fluent::Plugin::ElasticsearchOutput::ConnectionFailure) do
398
- driver(config)
399
- end
400
-
401
- assert_equal(connection_resets, 4)
402
- end
403
-
404
- def test_templates_create
405
- cwd = File.dirname(__FILE__)
406
- template_file = File.join(cwd, 'test_template.json')
407
- config = %{
408
- host logs.google.com
409
- port 777
410
- scheme https
411
- path /es/
412
- user john
413
- password doe
414
- templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
415
- }
416
-
417
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
418
- to_return(:status => 200, :body => "", :headers => {})
419
- # check if template exists
420
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
421
- to_return(:status => 404, :body => "", :headers => {})
422
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
423
- to_return(:status => 404, :body => "", :headers => {})
424
-
425
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
426
- to_return(:status => 200, :body => "", :headers => {}) #exists
427
-
428
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
429
- to_return(:status => 200, :body => "", :headers => {})
430
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
431
- to_return(:status => 200, :body => "", :headers => {})
432
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
433
- to_return(:status => 200, :body => "", :headers => {})
434
-
435
- driver(config)
436
-
437
- assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
438
- assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
439
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
440
- end
441
-
442
- def test_templates_overwrite
443
- cwd = File.dirname(__FILE__)
444
- template_file = File.join(cwd, 'test_template.json')
445
- config = %{
446
- host logs.google.com
447
- port 777
448
- scheme https
449
- path /es/
450
- user john
451
- password doe
452
- templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
453
- template_overwrite true
454
- }
455
-
456
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
457
- to_return(:status => 200, :body => "", :headers => {})
458
- # check if template exists
459
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
460
- to_return(:status => 200, :body => "", :headers => {})
461
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
462
- to_return(:status => 200, :body => "", :headers => {})
463
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
464
- to_return(:status => 200, :body => "", :headers => {}) #exists
465
-
466
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
467
- to_return(:status => 200, :body => "", :headers => {})
468
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
469
- to_return(:status => 200, :body => "", :headers => {})
470
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
471
- to_return(:status => 200, :body => "", :headers => {})
472
-
473
- driver(config)
474
-
475
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
476
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
477
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
478
- end
479
-
480
- def test_templates_not_used
481
- cwd = File.dirname(__FILE__)
482
- template_file = File.join(cwd, 'test_template.json')
483
-
484
- config = %{
485
- host logs.google.com
486
- port 777
487
- scheme https
488
- path /es/
489
- user john
490
- password doe
491
- template_name logstash
492
- template_file #{template_file}
493
- templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
494
- }
495
- # connection start
496
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
497
- to_return(:status => 200, :body => "", :headers => {})
498
- # check if template exists
499
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
500
- to_return(:status => 404, :body => "", :headers => {})
501
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
502
- to_return(:status => 404, :body => "", :headers => {})
503
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
504
- to_return(:status => 404, :body => "", :headers => {})
505
- #creation
506
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
507
- to_return(:status => 200, :body => "", :headers => {})
508
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
509
- to_return(:status => 200, :body => "", :headers => {})
510
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
511
- to_return(:status => 200, :body => "", :headers => {})
512
-
513
- driver(config)
514
-
515
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
516
-
517
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1")
518
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
519
- end
520
-
521
- def test_templates_can_be_partially_created_if_error_occurs
522
- cwd = File.dirname(__FILE__)
523
- template_file = File.join(cwd, 'test_template.json')
524
- config = %{
525
- host logs.google.com
526
- port 777
527
- scheme https
528
- path /es/
529
- user john
530
- password doe
531
- templates {"logstash1":"#{template_file}", "logstash2":"/abc" }
532
- }
533
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
534
- to_return(:status => 200, :body => "", :headers => {})
535
- # check if template exists
536
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
537
- to_return(:status => 404, :body => "", :headers => {})
538
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
539
- to_return(:status => 404, :body => "", :headers => {})
540
-
541
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
542
- to_return(:status => 200, :body => "", :headers => {})
543
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
544
- to_return(:status => 200, :body => "", :headers => {})
545
-
546
- assert_raise(RuntimeError) {
547
- driver(config)
548
- }
549
-
550
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
551
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
552
- end
553
-
554
- def test_legacy_hosts_list
555
- config = %{
556
- hosts host1:50,host2:100,host3
557
- scheme https
558
- path /es/
559
- port 123
560
- }
561
- instance = driver(config).instance
562
-
563
- assert_equal 3, instance.get_connection_options[:hosts].length
564
- host1, host2, host3 = instance.get_connection_options[:hosts]
565
-
566
- assert_equal 'host1', host1[:host]
567
- assert_equal 50, host1[:port]
568
- assert_equal 'https', host1[:scheme]
569
- assert_equal '/es/', host2[:path]
570
- assert_equal 'host3', host3[:host]
571
- assert_equal 123, host3[:port]
572
- assert_equal 'https', host3[:scheme]
573
- assert_equal '/es/', host3[:path]
574
- end
575
-
576
- def test_hosts_list
577
- config = %{
578
- hosts https://john:password@host1:443/elastic/,http://host2
579
- path /default_path
580
- user default_user
581
- password default_password
582
- }
583
- instance = driver(config).instance
584
-
585
- assert_equal 2, instance.get_connection_options[:hosts].length
586
- host1, host2 = instance.get_connection_options[:hosts]
587
-
588
- assert_equal 'host1', host1[:host]
589
- assert_equal 443, host1[:port]
590
- assert_equal 'https', host1[:scheme]
591
- assert_equal 'john', host1[:user]
592
- assert_equal 'password', host1[:password]
593
- assert_equal '/elastic/', host1[:path]
594
-
595
- assert_equal 'host2', host2[:host]
596
- assert_equal 'http', host2[:scheme]
597
- assert_equal 'default_user', host2[:user]
598
- assert_equal 'default_password', host2[:password]
599
- assert_equal '/default_path', host2[:path]
600
- end
601
-
602
- def test_hosts_list_with_escape_placeholders
603
- config = %{
604
- hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
605
- path /default_path
606
- user default_user
607
- password default_password
608
- }
609
- instance = driver(config).instance
610
-
611
- assert_equal 2, instance.get_connection_options[:hosts].length
612
- host1, host2 = instance.get_connection_options[:hosts]
613
-
614
- assert_equal 'host1', host1[:host]
615
- assert_equal 443, host1[:port]
616
- assert_equal 'https', host1[:scheme]
617
- assert_equal 'j%2Bhn', host1[:user]
618
- assert_equal 'passw%40rd', host1[:password]
619
- assert_equal '/elastic/', host1[:path]
620
-
621
- assert_equal 'host2', host2[:host]
622
- assert_equal 'http', host2[:scheme]
623
- assert_equal 'default_user', host2[:user]
624
- assert_equal 'default_password', host2[:password]
625
- assert_equal '/default_path', host2[:path]
626
- end
627
-
628
- def test_single_host_params_and_defaults
629
- config = %{
630
- host logs.google.com
631
- user john
632
- password doe
633
- }
634
- instance = driver(config).instance
635
-
636
- assert_equal 1, instance.get_connection_options[:hosts].length
637
- host1 = instance.get_connection_options[:hosts][0]
638
-
639
- assert_equal 'logs.google.com', host1[:host]
640
- assert_equal 9200, host1[:port]
641
- assert_equal 'http', host1[:scheme]
642
- assert_equal 'john', host1[:user]
643
- assert_equal 'doe', host1[:password]
644
- assert_equal nil, host1[:path]
645
- end
646
-
647
- def test_single_host_params_and_defaults_with_escape_placeholders
648
- config = %{
649
- host logs.google.com
650
- user %{j+hn}
651
- password %{d@e}
652
- }
653
- instance = driver(config).instance
654
-
655
- assert_equal 1, instance.get_connection_options[:hosts].length
656
- host1 = instance.get_connection_options[:hosts][0]
657
-
658
- assert_equal 'logs.google.com', host1[:host]
659
- assert_equal 9200, host1[:port]
660
- assert_equal 'http', host1[:scheme]
661
- assert_equal 'j%2Bhn', host1[:user]
662
- assert_equal 'd%40e', host1[:password]
663
- assert_equal nil, host1[:path]
664
- end
665
-
666
- def test_content_type_header
667
- stub_request(:head, "http://localhost:9200/").
668
- to_return(:status => 200, :body => "", :headers => {})
669
- if Elasticsearch::VERSION >= "6.0.2"
670
- elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
671
- with(headers: { "Content-Type" => "application/x-ndjson" })
672
- else
673
- elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
674
- with(headers: { "Content-Type" => "application/json" })
675
- end
676
- driver.run(default_tag: 'test') do
677
- driver.feed(sample_record)
678
- end
679
- assert_requested(elastic_request)
680
- end
681
-
682
- def test_write_message_with_bad_chunk
683
- driver.configure("target_index_key bad_value\n@log_level debug\n")
684
- stub_elastic_ping
685
- stub_elastic
686
- driver.run(default_tag: 'test') do
687
- driver.feed({'bad_value'=>"\255"})
688
- end
689
- error_log = driver.error_events.map {|e| e.last.message }
690
-
691
- assert_logs_include(error_log, /(input string invalid)|(invalid byte sequence in UTF-8)/)
692
- end
693
-
694
- def test_writes_to_default_index
695
- stub_elastic_ping
696
- stub_elastic
697
- driver.run(default_tag: 'test') do
698
- driver.feed(sample_record)
699
- end
700
- assert_equal('fluentd', index_cmds.first['index']['_index'])
701
- end
702
-
703
- def test_writes_to_default_type
704
- stub_elastic_ping
705
- stub_elastic
706
- driver.run(default_tag: 'test') do
707
- driver.feed(sample_record)
708
- end
709
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
710
- end
711
-
712
- def test_writes_to_speficied_index
713
- driver.configure("index_name myindex\n")
714
- stub_elastic_ping
715
- stub_elastic
716
- driver.run(default_tag: 'test') do
717
- driver.feed(sample_record)
718
- end
719
- assert_equal('myindex', index_cmds.first['index']['_index'])
720
- end
721
-
722
- class IndexNamePlaceholdersTest < self
723
- def test_writes_to_speficied_index_with_tag_placeholder
724
- driver.configure("index_name myindex.${tag}\n")
725
- stub_elastic_ping
726
- stub_elastic
727
- driver.run(default_tag: 'test') do
728
- driver.feed(sample_record)
729
- end
730
- assert_equal('myindex.test', index_cmds.first['index']['_index'])
731
- end
732
-
733
- def test_writes_to_speficied_index_with_time_placeholder
734
- driver.configure(Fluent::Config::Element.new(
735
- 'ROOT', '', {
736
- '@type' => 'elasticsearch',
737
- 'index_name' => 'myindex.%Y.%m.%d',
738
- }, [
739
- Fluent::Config::Element.new('buffer', 'tag,time', {
740
- 'chunk_keys' => ['tag', 'time'],
741
- 'timekey' => 3600,
742
- }, [])
743
- ]
744
- ))
745
- stub_elastic_ping
746
- stub_elastic
747
- time = Time.parse Date.today.iso8601
748
- driver.run(default_tag: 'test') do
749
- driver.feed(time.to_i, sample_record)
750
- end
751
- assert_equal("myindex.#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['index']['_index'])
752
- end
753
-
754
- def test_writes_to_speficied_index_with_custom_key_placeholder
755
- driver.configure(Fluent::Config::Element.new(
756
- 'ROOT', '', {
757
- '@type' => 'elasticsearch',
758
- 'index_name' => 'myindex.${pipeline_id}',
759
- }, [
760
- Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
761
- ]
762
- ))
763
- time = Time.parse Date.today.iso8601
764
- pipeline_id = "mypipeline"
765
- logstash_index = "myindex.#{pipeline_id}"
766
- stub_elastic_ping
767
- stub_elastic
768
- driver.run(default_tag: 'test') do
769
- driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
770
- end
771
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
772
- end
773
- end
774
-
775
- def test_writes_to_speficied_index_uppercase
776
- driver.configure("index_name MyIndex\n")
777
- stub_elastic_ping
778
- stub_elastic
779
- driver.run(default_tag: 'test') do
780
- driver.feed(sample_record)
781
- end
782
- # Allthough index_name has upper-case characters,
783
- # it should be set as lower-case when sent to elasticsearch.
784
- assert_equal('myindex', index_cmds.first['index']['_index'])
785
- end
786
-
787
- def test_writes_to_target_index_key
788
- driver.configure("target_index_key @target_index\n")
789
- stub_elastic_ping
790
- stub_elastic
791
- record = sample_record.clone
792
- driver.run(default_tag: 'test') do
793
- driver.feed(sample_record.merge('@target_index' => 'local-override'))
794
- end
795
- assert_equal('local-override', index_cmds.first['index']['_index'])
796
- assert_nil(index_cmds[1]['@target_index'])
797
- end
798
-
799
- def test_writes_to_target_index_key_logstash
800
- driver.configure("target_index_key @target_index
801
- logstash_format true")
802
- time = Time.parse Date.today.iso8601
803
- stub_elastic_ping
804
- stub_elastic
805
- driver.run(default_tag: 'test') do
806
- driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
807
- end
808
- assert_equal('local-override', index_cmds.first['index']['_index'])
809
- end
810
-
811
- def test_writes_to_target_index_key_logstash_uppercase
812
- driver.configure("target_index_key @target_index
813
- logstash_format true")
814
- time = Time.parse Date.today.iso8601
815
- stub_elastic_ping
816
- stub_elastic
817
- driver.run(default_tag: 'test') do
818
- driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
819
- end
820
- # Allthough @target_index has upper-case characters,
821
- # it should be set as lower-case when sent to elasticsearch.
822
- assert_equal('local-override', index_cmds.first['index']['_index'])
823
- end
824
-
825
- def test_writes_to_default_index_with_pipeline
826
- pipeline = "fluentd"
827
- driver.configure("pipeline #{pipeline}")
828
- stub_elastic_ping
829
- stub_elastic
830
- driver.run(default_tag: 'test') do
831
- driver.feed(sample_record)
832
- end
833
- assert_equal(pipeline, index_cmds.first['index']['pipeline'])
834
- end
835
-
836
- def test_writes_to_target_index_key_fallack
837
- driver.configure("target_index_key @target_index\n")
838
- stub_elastic_ping
839
- stub_elastic
840
- driver.run(default_tag: 'test') do
841
- driver.feed(sample_record)
842
- end
843
- assert_equal('fluentd', index_cmds.first['index']['_index'])
844
- end
845
-
846
- def test_writes_to_target_index_key_fallack_logstash
847
- driver.configure("target_index_key @target_index\n
848
- logstash_format true")
849
- time = Time.parse Date.today.iso8601
850
- logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
851
- stub_elastic_ping
852
- stub_elastic
853
- driver.run(default_tag: 'test') do
854
- driver.feed(time.to_i, sample_record)
855
- end
856
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
857
- end
858
-
859
- data("border" => {"es_version" => 6, "_type" => "mytype"},
860
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
861
- )
862
- def test_writes_to_speficied_type(data)
863
- driver('', data["es_version"]).configure("type_name mytype\n")
864
- stub_elastic_ping
865
- stub_elastic
866
- driver.run(default_tag: 'test') do
867
- driver.feed(sample_record)
868
- end
869
- assert_equal(data['_type'], index_cmds.first['index']['_type'])
870
- end
871
-
872
- data("border" => {"es_version" => 6, "_type" => "mytype.test"},
873
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
874
- )
875
- def test_writes_to_speficied_type_with_placeholders(data)
876
- driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
877
- stub_elastic_ping
878
- stub_elastic
879
- driver.run(default_tag: 'test') do
880
- driver.feed(sample_record)
881
- end
882
- assert_equal(data['_type'], index_cmds.first['index']['_type'])
883
- end
884
-
885
- data("old" => {"es_version" => 2, "_type" => "local-override"},
886
- "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
887
- "border" => {"es_version" => 6, "_type" => "fluentd"},
888
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
889
- )
890
- def test_writes_to_target_type_key(data)
891
- driver('', data["es_version"]).configure("target_type_key @target_type\n")
892
- stub_elastic_ping
893
- stub_elastic
894
- record = sample_record.clone
895
- driver.run(default_tag: 'test') do
896
- driver.feed(sample_record.merge('@target_type' => 'local-override'))
897
- end
898
- assert_equal(data["_type"], index_cmds.first['index']['_type'])
899
- assert_nil(index_cmds[1]['@target_type'])
900
- end
901
-
902
- def test_writes_to_target_type_key_fallack_to_default
903
- driver.configure("target_type_key @target_type\n")
904
- stub_elastic_ping
905
- stub_elastic
906
- driver.run(default_tag: 'test') do
907
- driver.feed(sample_record)
908
- end
909
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
910
- end
911
-
912
- def test_writes_to_target_type_key_fallack_to_type_name
913
- driver.configure("target_type_key @target_type
914
- type_name mytype")
915
- stub_elastic_ping
916
- stub_elastic
917
- driver.run(default_tag: 'test') do
918
- driver.feed(sample_record)
919
- end
920
- assert_equal('mytype', index_cmds.first['index']['_type'])
921
- end
922
-
923
- data("old" => {"es_version" => 2, "_type" => "local-override"},
924
- "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
925
- "border" => {"es_version" => 6, "_type" => "fluentd"},
926
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
927
- )
928
- def test_writes_to_target_type_key_nested(data)
929
- driver('', data["es_version"]).configure("target_type_key kubernetes.labels.log_type\n")
930
- stub_elastic_ping
931
- stub_elastic
932
- driver.run(default_tag: 'test') do
933
- driver.feed(sample_record.merge('kubernetes' => {
934
- 'labels' => {
935
- 'log_type' => 'local-override'
936
- }
937
- }))
938
- end
939
- assert_equal(data["_type"], index_cmds.first['index']['_type'])
940
- assert_nil(index_cmds[1]['kubernetes']['labels']['log_type'])
941
- end
942
-
943
- def test_writes_to_target_type_key_fallack_to_default_nested
944
- driver.configure("target_type_key kubernetes.labels.log_type\n")
945
- stub_elastic_ping
946
- stub_elastic
947
- driver.run(default_tag: 'test') do
948
- driver.feed(sample_record.merge('kubernetes' => {
949
- 'labels' => {
950
- 'other_labels' => 'test'
951
- }
952
- }))
953
- end
954
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
955
- end
956
-
957
- def test_writes_to_speficied_host
958
- driver.configure("host 192.168.33.50\n")
959
- stub_elastic_ping("http://192.168.33.50:9200")
960
- elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
961
- driver.run(default_tag: 'test') do
962
- driver.feed(sample_record)
963
- end
964
- assert_requested(elastic_request)
965
- end
966
-
967
- def test_writes_to_speficied_port
968
- driver.configure("port 9201\n")
969
- stub_elastic_ping("http://localhost:9201")
970
- elastic_request = stub_elastic("http://localhost:9201/_bulk")
971
- driver.run(default_tag: 'test') do
972
- driver.feed(sample_record)
973
- end
974
- assert_requested(elastic_request)
975
- end
976
-
977
- def test_writes_to_multi_hosts
978
- hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
979
- hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
980
-
981
- driver.configure("hosts #{hosts_string}")
982
-
983
- hosts.each do |host_info|
984
- host, port = host_info
985
- stub_elastic_ping("http://#{host}:#{port}")
986
- stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
987
- end
988
-
989
- driver.run(default_tag: 'test') do
990
- 1000.times do
991
- driver.feed(sample_record.merge('age'=>rand(100)))
992
- end
993
- end
994
-
995
- # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
996
- # it's Fluentd test driver's constraint
997
- # so @index_command_counts.size is always 1
998
-
999
- assert(@index_command_counts.size > 0, "not working with hosts options")
1000
-
1001
- total = 0
1002
- @index_command_counts.each do |url, count|
1003
- total += count
1004
- end
1005
- assert_equal(2000, total)
1006
- end
1007
-
1008
- def test_nested_record_with_flattening_on
1009
- driver.configure("flatten_hashes true
1010
- flatten_hashes_separator |")
1011
-
1012
- original_hash = {"foo" => {"bar" => "baz"}, "people" => [
1013
- {"age" => "25", "height" => "1ft"},
1014
- {"age" => "30", "height" => "2ft"}
1015
- ]}
1016
-
1017
- expected_output = {"foo|bar"=>"baz", "people" => [
1018
- {"age" => "25", "height" => "1ft"},
1019
- {"age" => "30", "height" => "2ft"}
1020
- ]}
1021
-
1022
- stub_elastic_ping
1023
- stub_elastic
1024
- driver.run(default_tag: 'test') do
1025
- driver.feed(original_hash)
1026
- end
1027
- assert_equal expected_output, index_cmds[1]
1028
- end
1029
-
1030
- def test_nested_record_with_flattening_off
1031
- # flattening off by default
1032
-
1033
- original_hash = {"foo" => {"bar" => "baz"}}
1034
- expected_output = {"foo" => {"bar" => "baz"}}
1035
-
1036
- stub_elastic_ping
1037
- stub_elastic
1038
- driver.run(default_tag: 'test') do
1039
- driver.feed(original_hash)
1040
- end
1041
- assert_equal expected_output, index_cmds[1]
1042
- end
1043
-
1044
- def test_makes_bulk_request
1045
- stub_elastic_ping
1046
- stub_elastic
1047
- driver.run(default_tag: 'test') do
1048
- driver.feed(sample_record)
1049
- driver.feed(sample_record.merge('age' => 27))
1050
- end
1051
- assert_equal(4, index_cmds.count)
1052
- end
1053
-
1054
- def test_all_records_are_preserved_in_bulk
1055
- stub_elastic_ping
1056
- stub_elastic
1057
- driver.run(default_tag: 'test') do
1058
- driver.feed(sample_record)
1059
- driver.feed(sample_record.merge('age' => 27))
1060
- end
1061
- assert_equal(26, index_cmds[1]['age'])
1062
- assert_equal(27, index_cmds[3]['age'])
1063
- end
1064
-
1065
- def test_writes_to_logstash_index
1066
- driver.configure("logstash_format true\n")
1067
- #
1068
- # This is 1 second past midnight in BST, so the UTC index should be the day before
1069
- dt = DateTime.new(2015, 6, 1, 0, 0, 1, "+01:00")
1070
- logstash_index = "logstash-2015.05.31"
1071
- stub_elastic_ping
1072
- stub_elastic
1073
- driver.run(default_tag: 'test') do
1074
- driver.feed(dt.to_time.to_i, sample_record)
1075
- end
1076
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1077
- end
1078
-
1079
- def test_writes_to_logstash_non_utc_index
1080
- driver.configure("logstash_format true
1081
- utc_index false")
1082
- # When using `utc_index false` the index time will be the local day of
1083
- # ingestion time
1084
- time = Date.today.to_time
1085
- index = "logstash-#{time.strftime("%Y.%m.%d")}"
1086
- stub_elastic_ping
1087
- stub_elastic
1088
- driver.run(default_tag: 'test') do
1089
- driver.feed(time.to_i, sample_record)
1090
- end
1091
- assert_equal(index, index_cmds.first['index']['_index'])
1092
- end
1093
-
1094
- def test_writes_to_logstash_index_with_specified_prefix
1095
- driver.configure("logstash_format true
1096
- logstash_prefix myprefix")
1097
- time = Time.parse Date.today.iso8601
1098
- logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1099
- stub_elastic_ping
1100
- stub_elastic
1101
- driver.run(default_tag: 'test') do
1102
- driver.feed(time.to_i, sample_record)
1103
- end
1104
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1105
- end
1106
-
1107
- def test_writes_to_logstash_index_with_specified_prefix_and_separator
1108
- separator = '_'
1109
- driver.configure("logstash_format true
1110
- logstash_prefix_separator #{separator}
1111
- logstash_prefix myprefix")
1112
- time = Time.parse Date.today.iso8601
1113
- logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
1114
- stub_elastic_ping
1115
- stub_elastic
1116
- driver.run(default_tag: 'test') do
1117
- driver.feed(time.to_i, sample_record)
1118
- end
1119
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1120
- end
1121
-
1122
- class LogStashPrefixPlaceholdersTest < self
1123
- def test_writes_to_logstash_index_with_specified_prefix_and_tag_placeholder
1124
- driver.configure("logstash_format true
1125
- logstash_prefix myprefix-${tag}")
1126
- time = Time.parse Date.today.iso8601
1127
- logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
1128
- stub_elastic_ping
1129
- stub_elastic
1130
- driver.run(default_tag: 'test') do
1131
- driver.feed(time.to_i, sample_record)
1132
- end
1133
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1134
- end
1135
-
1136
- def test_writes_to_logstash_index_with_specified_prefix_and_time_placeholder
1137
- driver.configure(Fluent::Config::Element.new(
1138
- 'ROOT', '', {
1139
- '@type' => 'elasticsearch',
1140
- 'logstash_format' => true,
1141
- 'logstash_prefix' => 'myprefix-%H',
1142
- }, [
1143
- Fluent::Config::Element.new('buffer', 'tag,time', {
1144
- 'chunk_keys' => ['tag', 'time'],
1145
- 'timekey' => 3600,
1146
- }, [])
1147
- ]
1148
- ))
1149
- time = Time.parse Date.today.iso8601
1150
- logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
1151
- stub_elastic_ping
1152
- stub_elastic
1153
- driver.run(default_tag: 'test') do
1154
- driver.feed(time.to_i, sample_record)
1155
- end
1156
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1157
- end
1158
-
1159
- def test_writes_to_logstash_index_with_specified_prefix_and_custom_key_placeholder
1160
- driver.configure(Fluent::Config::Element.new(
1161
- 'ROOT', '', {
1162
- '@type' => 'elasticsearch',
1163
- 'logstash_format' => true,
1164
- 'logstash_prefix' => 'myprefix-${pipeline_id}',
1165
- }, [
1166
- Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
1167
- ]
1168
- ))
1169
- time = Time.parse Date.today.iso8601
1170
- pipeline_id = "mypipeline"
1171
- logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
1172
- stub_elastic_ping
1173
- stub_elastic
1174
- driver.run(default_tag: 'test') do
1175
- driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
1176
- end
1177
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1178
- end
1179
- end
1180
-
1181
- def test_writes_to_logstash_index_with_specified_prefix_uppercase
1182
- driver.configure("logstash_format true
1183
- logstash_prefix MyPrefix")
1184
- time = Time.parse Date.today.iso8601
1185
- logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1186
- stub_elastic_ping
1187
- stub_elastic
1188
- driver.run(default_tag: 'test') do
1189
- driver.feed(time.to_i, sample_record)
1190
- end
1191
- # Allthough logstash_prefix has upper-case characters,
1192
- # it should be set as lower-case when sent to elasticsearch.
1193
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1194
- end
1195
-
1196
- def test_writes_to_logstash_index_with_specified_dateformat
1197
- driver.configure("logstash_format true
1198
- logstash_dateformat %Y.%m")
1199
- time = Time.parse Date.today.iso8601
1200
- logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
1201
- stub_elastic_ping
1202
- stub_elastic
1203
- driver.run(default_tag: 'test') do
1204
- driver.feed(time.to_i, sample_record)
1205
- end
1206
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1207
- end
1208
-
1209
- def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
1210
- driver.configure("logstash_format true
1211
- logstash_prefix myprefix
1212
- logstash_dateformat %Y.%m")
1213
- time = Time.parse Date.today.iso8601
1214
- logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
1215
- stub_elastic_ping
1216
- stub_elastic
1217
- driver.run(default_tag: 'test') do
1218
- driver.feed(time.to_i, sample_record)
1219
- end
1220
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1221
- end
1222
-
1223
- def test_error_if_tag_not_in_chunk_keys
1224
- assert_raise(Fluent::ConfigError) {
1225
- config = %{
1226
- <buffer foo>
1227
- </buffer>
1228
- }
1229
- driver.configure(config)
1230
- }
1231
- end
1232
-
1233
- def test_can_use_custom_chunk_along_with_tag
1234
- config = %{
1235
- <buffer tag, foo>
1236
- </buffer>
1237
- }
1238
- driver.configure(config)
1239
- end
1240
-
1241
- def test_doesnt_add_logstash_timestamp_by_default
1242
- stub_elastic_ping
1243
- stub_elastic
1244
- driver.run(default_tag: 'test') do
1245
- driver.feed(sample_record)
1246
- end
1247
- assert_nil(index_cmds[1]['@timestamp'])
1248
- end
1249
-
1250
- def test_adds_timestamp_when_logstash
1251
- driver.configure("logstash_format true\n")
1252
- stub_elastic_ping
1253
- stub_elastic
1254
- ts = DateTime.now
1255
- time = Fluent::EventTime.from_time(ts.to_time)
1256
- driver.run(default_tag: 'test') do
1257
- driver.feed(time, sample_record)
1258
- end
1259
- assert(index_cmds[1].has_key? '@timestamp')
1260
- assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1261
- end
1262
-
1263
- def test_adds_timestamp_when_include_timestamp
1264
- driver.configure("include_timestamp true\n")
1265
- stub_elastic_ping
1266
- stub_elastic
1267
- ts = DateTime.now
1268
- time = Fluent::EventTime.from_time(ts.to_time)
1269
- driver.run(default_tag: 'test') do
1270
- driver.feed(time, sample_record)
1271
- end
1272
- assert(index_cmds[1].has_key? '@timestamp')
1273
- assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1274
- end
1275
-
1276
- def test_uses_custom_timestamp_when_included_in_record
1277
- driver.configure("logstash_format true\n")
1278
- stub_elastic_ping
1279
- stub_elastic
1280
- ts = DateTime.new(2001,2,3).iso8601
1281
- driver.run(default_tag: 'test') do
1282
- driver.feed(sample_record.merge!('@timestamp' => ts))
1283
- end
1284
- assert(index_cmds[1].has_key? '@timestamp')
1285
- assert_equal(index_cmds[1]['@timestamp'], ts)
1286
- end
1287
-
1288
- def test_uses_custom_timestamp_when_included_in_record_without_logstash
1289
- driver.configure("include_timestamp true\n")
1290
- stub_elastic_ping
1291
- stub_elastic
1292
- ts = DateTime.new(2001,2,3).iso8601
1293
- driver.run(default_tag: 'test') do
1294
- driver.feed(sample_record.merge!('@timestamp' => ts))
1295
- end
1296
- assert(index_cmds[1].has_key? '@timestamp')
1297
- assert_equal(index_cmds[1]['@timestamp'], ts)
1298
- end
1299
-
1300
- def test_uses_custom_time_key
1301
- driver.configure("logstash_format true
1302
- time_key vtm\n")
1303
- stub_elastic_ping
1304
- stub_elastic
1305
- ts = DateTime.new(2001,2,3).iso8601(9)
1306
- driver.run(default_tag: 'test') do
1307
- driver.feed(sample_record.merge!('vtm' => ts))
1308
- end
1309
- assert(index_cmds[1].has_key? '@timestamp')
1310
- assert_equal(index_cmds[1]['@timestamp'], ts)
1311
- end
1312
-
1313
- def test_uses_custom_time_key_with_format
1314
- driver.configure("logstash_format true
1315
- time_key_format %Y-%m-%d %H:%M:%S.%N%z
1316
- time_key vtm\n")
1317
- stub_elastic_ping
1318
- stub_elastic
1319
- ts = "2001-02-03 13:14:01.673+02:00"
1320
- driver.run(default_tag: 'test') do
1321
- driver.feed(sample_record.merge!('vtm' => ts))
1322
- end
1323
- assert(index_cmds[1].has_key? '@timestamp')
1324
- assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1325
- assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1326
- end
1327
-
1328
- def test_uses_custom_time_key_with_format_without_logstash
1329
- driver.configure("include_timestamp true
1330
- index_name test
1331
- time_key_format %Y-%m-%d %H:%M:%S.%N%z
1332
- time_key vtm\n")
1333
- stub_elastic_ping
1334
- stub_elastic
1335
- ts = "2001-02-03 13:14:01.673+02:00"
1336
- driver.run(default_tag: 'test') do
1337
- driver.feed(sample_record.merge!('vtm' => ts))
1338
- end
1339
- assert(index_cmds[1].has_key? '@timestamp')
1340
- assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1341
- assert_equal("test", index_cmds[0]['index']['_index'])
1342
- end
1343
-
1344
- def test_uses_custom_time_key_exclude_timekey
1345
- driver.configure("logstash_format true
1346
- time_key vtm
1347
- time_key_exclude_timestamp true\n")
1348
- stub_elastic_ping
1349
- stub_elastic
1350
- ts = DateTime.new(2001,2,3).iso8601
1351
- driver.run(default_tag: 'test') do
1352
- driver.feed(sample_record.merge!('vtm' => ts))
1353
- end
1354
- assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be messing')
1355
- end
1356
-
1357
- def test_uses_custom_time_key_format
1358
- driver.configure("logstash_format true
1359
- time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1360
- stub_elastic_ping
1361
- stub_elastic
1362
- ts = "2001-02-03T13:14:01.673+02:00"
1363
- driver.run(default_tag: 'test') do
1364
- driver.feed(sample_record.merge!('@timestamp' => ts))
1365
- end
1366
- assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1367
- assert(index_cmds[1].has_key? '@timestamp')
1368
- assert_equal(index_cmds[1]['@timestamp'], ts)
1369
- end
1370
-
1371
- def test_uses_custom_time_key_format_without_logstash
1372
- driver.configure("include_timestamp true
1373
- index_name test
1374
- time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1375
- stub_elastic_ping
1376
- stub_elastic
1377
- ts = "2001-02-03T13:14:01.673+02:00"
1378
- driver.run(default_tag: 'test') do
1379
- driver.feed(sample_record.merge!('@timestamp' => ts))
1380
- end
1381
- assert_equal("test", index_cmds[0]['index']['_index'])
1382
- assert(index_cmds[1].has_key? '@timestamp')
1383
- assert_equal(index_cmds[1]['@timestamp'], ts)
1384
- end
1385
-
1386
- data(:default => nil,
1387
- :custom_tag => 'es_plugin.output.time.error')
1388
- def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
1389
- tag_config = tag_for_error ? "time_parse_error_tag #{tag_for_error}" : ''
1390
- tag_for_error = 'Fluent::ElasticsearchOutput::TimeParser.error' if tag_for_error.nil?
1391
- driver.configure("logstash_format true
1392
- time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n#{tag_config}\n")
1393
- stub_elastic_ping
1394
- stub_elastic
1395
-
1396
- ts = "2001/02/03 13:14:01,673+02:00"
1397
- index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
1398
-
1399
- flexmock(driver.instance.router).should_receive(:emit_error_event)
1400
- .with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once
1401
- driver.run(default_tag: 'test') do
1402
- driver.feed(sample_record.merge!('@timestamp' => ts))
1403
- end
1404
-
1405
- assert_equal(index, index_cmds[0]['index']['_index'])
1406
- assert(index_cmds[1].has_key? '@timestamp')
1407
- assert_equal(index_cmds[1]['@timestamp'], ts)
1408
- end
1409
-
1410
-
1411
- def test_uses_custom_time_key_format_obscure_format
1412
- driver.configure("logstash_format true
1413
- time_key_format %a %b %d %H:%M:%S %Z %Y\n")
1414
- stub_elastic_ping
1415
- stub_elastic
1416
- ts = "Thu Nov 29 14:33:20 GMT 2001"
1417
- driver.run(default_tag: 'test') do
1418
- driver.feed(sample_record.merge!('@timestamp' => ts))
1419
- end
1420
- assert_equal("logstash-2001.11.29", index_cmds[0]['index']['_index'])
1421
- assert(index_cmds[1].has_key? '@timestamp')
1422
- assert_equal(index_cmds[1]['@timestamp'], ts)
1423
- end
1424
-
1425
- def test_uses_nanosecond_precision_by_default
1426
- driver.configure("logstash_format true\n")
1427
- stub_elastic_ping
1428
- stub_elastic
1429
- time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1430
- driver.run(default_tag: 'test') do
1431
- driver.feed(time, sample_record)
1432
- end
1433
- assert(index_cmds[1].has_key? '@timestamp')
1434
- assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(9))
1435
- end
1436
-
1437
- def test_uses_subsecond_precision_when_configured
1438
- driver.configure("logstash_format true
1439
- time_precision 3\n")
1440
- stub_elastic_ping
1441
- stub_elastic
1442
- time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1443
- driver.run(default_tag: 'test') do
1444
- driver.feed(time, sample_record)
1445
- end
1446
- assert(index_cmds[1].has_key? '@timestamp')
1447
- assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(3))
1448
- end
1449
-
1450
- def test_doesnt_add_tag_key_by_default
1451
- stub_elastic_ping
1452
- stub_elastic
1453
- driver.run(default_tag: 'test') do
1454
- driver.feed(sample_record)
1455
- end
1456
- assert_nil(index_cmds[1]['tag'])
1457
- end
1458
-
1459
- def test_adds_tag_key_when_configured
1460
- driver.configure("include_tag_key true\n")
1461
- stub_elastic_ping
1462
- stub_elastic
1463
- driver.run(default_tag: 'mytag') do
1464
- driver.feed(sample_record)
1465
- end
1466
- assert(index_cmds[1].has_key?('tag'))
1467
- assert_equal(index_cmds[1]['tag'], 'mytag')
1468
- end
1469
-
1470
- def test_adds_id_key_when_configured
1471
- driver.configure("id_key request_id\n")
1472
- stub_elastic_ping
1473
- stub_elastic
1474
- driver.run(default_tag: 'test') do
1475
- driver.feed(sample_record)
1476
- end
1477
- assert_equal(index_cmds[0]['index']['_id'], '42')
1478
- end
1479
-
1480
- class NestedIdKeyTest < self
1481
- def test_adds_nested_id_key_with_dot
1482
- driver.configure("id_key nested.request_id\n")
1483
- stub_elastic_ping
1484
- stub_elastic
1485
- driver.run(default_tag: 'test') do
1486
- driver.feed(nested_sample_record)
1487
- end
1488
- assert_equal(index_cmds[0]['index']['_id'], '42')
1489
- end
1490
-
1491
- def test_adds_nested_id_key_with_dollar_dot
1492
- driver.configure("id_key $.nested.request_id\n")
1493
- stub_elastic_ping
1494
- stub_elastic
1495
- driver.run(default_tag: 'test') do
1496
- driver.feed(nested_sample_record)
1497
- end
1498
- assert_equal(index_cmds[0]['index']['_id'], '42')
1499
- end
1500
-
1501
- def test_adds_nested_id_key_with_bracket
1502
- driver.configure("id_key $['nested']['request_id']\n")
1503
- stub_elastic_ping
1504
- stub_elastic
1505
- driver.run(default_tag: 'test') do
1506
- driver.feed(nested_sample_record)
1507
- end
1508
- assert_equal(index_cmds[0]['index']['_id'], '42')
1509
- end
1510
- end
1511
-
1512
- def test_doesnt_add_id_key_if_missing_when_configured
1513
- driver.configure("id_key another_request_id\n")
1514
- stub_elastic_ping
1515
- stub_elastic
1516
- driver.run(default_tag: 'test') do
1517
- driver.feed(sample_record)
1518
- end
1519
- assert(!index_cmds[0]['index'].has_key?('_id'))
1520
- end
1521
-
1522
- def test_adds_id_key_when_not_configured
1523
- stub_elastic_ping
1524
- stub_elastic
1525
- driver.run(default_tag: 'test') do
1526
- driver.feed(sample_record)
1527
- end
1528
- assert(!index_cmds[0]['index'].has_key?('_id'))
1529
- end
1530
-
1531
- def test_adds_parent_key_when_configured
1532
- driver.configure("parent_key parent_id\n")
1533
- stub_elastic_ping
1534
- stub_elastic
1535
- driver.run(default_tag: 'test') do
1536
- driver.feed(sample_record)
1537
- end
1538
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1539
- end
1540
-
1541
- class NestedParentKeyTest < self
1542
- def test_adds_nested_parent_key_with_dot
1543
- driver.configure("parent_key nested.parent_id\n")
1544
- stub_elastic_ping
1545
- stub_elastic
1546
- driver.run(default_tag: 'test') do
1547
- driver.feed(nested_sample_record)
1548
- end
1549
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1550
- end
1551
-
1552
- def test_adds_nested_parent_key_with_dollar_dot
1553
- driver.configure("parent_key $.nested.parent_id\n")
1554
- stub_elastic_ping
1555
- stub_elastic
1556
- driver.run(default_tag: 'test') do
1557
- driver.feed(nested_sample_record)
1558
- end
1559
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1560
- end
1561
-
1562
- def test_adds_nested_parent_key_with_bracket
1563
- driver.configure("parent_key $['nested']['parent_id']\n")
1564
- stub_elastic_ping
1565
- stub_elastic
1566
- driver.run(default_tag: 'test') do
1567
- driver.feed(nested_sample_record)
1568
- end
1569
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1570
- end
1571
- end
1572
-
1573
- def test_doesnt_add_parent_key_if_missing_when_configured
1574
- driver.configure("parent_key another_parent_id\n")
1575
- stub_elastic_ping
1576
- stub_elastic
1577
- driver.run(default_tag: 'test') do
1578
- driver.feed(sample_record)
1579
- end
1580
- assert(!index_cmds[0]['index'].has_key?('_parent'))
1581
- end
1582
-
1583
- def test_adds_parent_key_when_not_configured
1584
- stub_elastic_ping
1585
- stub_elastic
1586
- driver.run(default_tag: 'test') do
1587
- driver.feed(sample_record)
1588
- end
1589
- assert(!index_cmds[0]['index'].has_key?('_parent'))
1590
- end
1591
-
1592
- def test_adds_routing_key_when_configured
1593
- driver.configure("routing_key routing_id\n")
1594
- stub_elastic_ping
1595
- stub_elastic
1596
- driver.run(default_tag: 'test') do
1597
- driver.feed(sample_record)
1598
- end
1599
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1600
- end
1601
-
1602
- class NestedRoutingKeyTest < self
1603
- def test_adds_nested_routing_key_with_dot
1604
- driver.configure("routing_key nested.routing_id\n")
1605
- stub_elastic_ping
1606
- stub_elastic
1607
- driver.run(default_tag: 'test') do
1608
- driver.feed(nested_sample_record)
1609
- end
1610
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1611
- end
1612
-
1613
- def test_adds_nested_routing_key_with_dollar_dot
1614
- driver.configure("routing_key $.nested.routing_id\n")
1615
- stub_elastic_ping
1616
- stub_elastic
1617
- driver.run(default_tag: 'test') do
1618
- driver.feed(nested_sample_record)
1619
- end
1620
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1621
- end
1622
-
1623
- def test_adds_nested_routing_key_with_bracket
1624
- driver.configure("routing_key $['nested']['routing_id']\n")
1625
- stub_elastic_ping
1626
- stub_elastic
1627
- driver.run(default_tag: 'test') do
1628
- driver.feed(nested_sample_record)
1629
- end
1630
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1631
- end
1632
- end
1633
-
1634
- def test_doesnt_add_routing_key_if_missing_when_configured
1635
- driver.configure("routing_key another_routing_id\n")
1636
- stub_elastic_ping
1637
- stub_elastic
1638
- driver.run(default_tag: 'test') do
1639
- driver.feed(sample_record)
1640
- end
1641
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1642
- end
1643
-
1644
- def test_adds_routing_key_when_not_configured
1645
- stub_elastic_ping
1646
- stub_elastic
1647
- driver.run(default_tag: 'test') do
1648
- driver.feed(sample_record)
1649
- end
1650
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1651
- end
1652
-
1653
- def test_remove_one_key
1654
- driver.configure("remove_keys key1\n")
1655
- stub_elastic_ping
1656
- stub_elastic
1657
- driver.run(default_tag: 'test') do
1658
- driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1659
- end
1660
- assert(!index_cmds[1].has_key?('key1'))
1661
- assert(index_cmds[1].has_key?('key2'))
1662
- end
1663
-
1664
- def test_remove_multi_keys
1665
- driver.configure("remove_keys key1, key2\n")
1666
- stub_elastic_ping
1667
- stub_elastic
1668
- driver.run(default_tag: 'test') do
1669
- driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1670
- end
1671
- assert(!index_cmds[1].has_key?('key1'))
1672
- assert(!index_cmds[1].has_key?('key2'))
1673
- end
1674
-
1675
- def test_request_error
1676
- stub_elastic_ping
1677
- stub_elastic_unavailable
1678
- assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
1679
- driver.run(default_tag: 'test') do
1680
- driver.feed(sample_record)
1681
- end
1682
- }
1683
- end
1684
-
1685
- def test_connection_failed_retry
1686
- connection_resets = 0
1687
-
1688
- stub_elastic_ping(url="http://localhost:9200").with do |req|
1689
- connection_resets += 1
1690
- end
1691
-
1692
- stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1693
- raise Faraday::ConnectionFailed, "Test message"
1694
- end
1695
-
1696
- driver.run(default_tag: 'test') do
1697
- driver.feed(sample_record)
1698
- end
1699
- assert_equal(connection_resets, 3)
1700
- end
1701
-
1702
- def test_reconnect_on_error_enabled
1703
- connection_resets = 0
1704
-
1705
- stub_elastic_ping(url="http://localhost:9200").with do |req|
1706
- connection_resets += 1
1707
- end
1708
-
1709
- stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1710
- raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1711
- end
1712
-
1713
- driver.configure("reconnect_on_error true\n")
1714
-
1715
- assert_raise(ZeroDivisionError) {
1716
- driver.run(default_tag: 'test', shutdown: false) do
1717
- driver.feed(sample_record)
1718
- end
1719
- }
1720
-
1721
- assert_raise(Timeout::Error) {
1722
- driver.run(default_tag: 'test', shutdown: false) do
1723
- driver.feed(sample_record)
1724
- end
1725
- }
1726
- # FIXME: Consider keywords arguments in #run and how to test this later.
1727
- # Because v0.14 test driver does not have 1 to 1 correspondence between #run and #flush in tests.
1728
- assert_equal(connection_resets, 1)
1729
- end
1730
-
1731
- def test_reconnect_on_error_disabled
1732
- connection_resets = 0
1733
-
1734
- stub_elastic_ping(url="http://localhost:9200").with do |req|
1735
- connection_resets += 1
1736
- end
1737
-
1738
- stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1739
- raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1740
- end
1741
-
1742
- driver.configure("reconnect_on_error false\n")
1743
-
1744
- assert_raise(ZeroDivisionError) {
1745
- driver.run(default_tag: 'test', shutdown: false) do
1746
- driver.feed(sample_record)
1747
- end
1748
- }
1749
-
1750
- assert_raise(Timeout::Error) {
1751
- driver.run(default_tag: 'test', shutdown: false) do
1752
- driver.feed(sample_record)
1753
- end
1754
- }
1755
- assert_equal(connection_resets, 1)
1756
- end
1757
-
1758
- def test_bulk_error_retags_when_configured
1759
- driver.configure("retry_tag retry\n")
1760
- stub_elastic_ping
1761
- stub_request(:post, 'http://localhost:9200/_bulk')
1762
- .to_return(lambda do |req|
1763
- { :status => 200,
1764
- :headers => { 'Content-Type' => 'json' },
1765
- :body => %({
1766
- "took" : 1,
1767
- "errors" : true,
1768
- "items" : [
1769
- {
1770
- "create" : {
1771
- "_index" : "foo",
1772
- "_type" : "bar",
1773
- "_id" : "abc",
1774
- "status" : 500,
1775
- "error" : {
1776
- "type" : "some unrecognized type",
1777
- "reason":"some error to cause version mismatch"
1778
- }
1779
- }
1780
- }
1781
- ]
1782
- })
1783
- }
1784
- end)
1785
-
1786
- driver.run(default_tag: 'test') do
1787
- driver.feed(1, sample_record)
1788
- end
1789
-
1790
- assert_equal [['retry', 1, sample_record]], driver.events
1791
- end
1792
-
1793
- def test_bulk_error
1794
- stub_elastic_ping
1795
- stub_request(:post, 'http://localhost:9200/_bulk')
1796
- .to_return(lambda do |req|
1797
- { :status => 200,
1798
- :headers => { 'Content-Type' => 'json' },
1799
- :body => %({
1800
- "took" : 1,
1801
- "errors" : true,
1802
- "items" : [
1803
- {
1804
- "create" : {
1805
- "_index" : "foo",
1806
- "_type" : "bar",
1807
- "_id" : "abc",
1808
- "status" : 500,
1809
- "error" : {
1810
- "type" : "some unrecognized type",
1811
- "reason":"some error to cause version mismatch"
1812
- }
1813
- }
1814
- },
1815
- {
1816
- "create" : {
1817
- "_index" : "foo",
1818
- "_type" : "bar",
1819
- "_id" : "abc",
1820
- "status" : 201
1821
- }
1822
- },
1823
- {
1824
- "create" : {
1825
- "_index" : "foo",
1826
- "_type" : "bar",
1827
- "_id" : "abc",
1828
- "status" : 500,
1829
- "error" : {
1830
- "type" : "some unrecognized type",
1831
- "reason":"some error to cause version mismatch"
1832
- }
1833
- }
1834
- },
1835
- {
1836
- "create" : {
1837
- "_index" : "foo",
1838
- "_type" : "bar",
1839
- "_id" : "abc",
1840
- "_id" : "abc",
1841
- "status" : 409
1842
- }
1843
- }
1844
- ]
1845
- })
1846
- }
1847
- end)
1848
-
1849
- driver.run(default_tag: 'test') do
1850
- driver.feed(1, sample_record)
1851
- driver.feed(2, sample_record)
1852
- driver.feed(3, sample_record)
1853
- driver.feed(4, sample_record)
1854
- end
1855
-
1856
- expect = [['test', 1, sample_record],
1857
- ['test', 3, sample_record]]
1858
- assert_equal expect, driver.events
1859
- end
1860
-
1861
- def test_update_should_not_write_if_theres_no_id
1862
- driver.configure("write_operation update\n")
1863
- stub_elastic_ping
1864
- stub_elastic
1865
- driver.run(default_tag: 'test') do
1866
- driver.feed(sample_record)
1867
- end
1868
- assert_nil(index_cmds)
1869
- end
1870
-
1871
- def test_upsert_should_not_write_if_theres_no_id
1872
- driver.configure("write_operation upsert\n")
1873
- stub_elastic_ping
1874
- stub_elastic
1875
- driver.run(default_tag: 'test') do
1876
- driver.feed(sample_record)
1877
- end
1878
- assert_nil(index_cmds)
1879
- end
1880
-
1881
- def test_create_should_not_write_if_theres_no_id
1882
- driver.configure("write_operation create\n")
1883
- stub_elastic_ping
1884
- stub_elastic
1885
- driver.run(default_tag: 'test') do
1886
- driver.feed(sample_record)
1887
- end
1888
- assert_nil(index_cmds)
1889
- end
1890
-
1891
- def test_update_should_write_update_op_and_doc_as_upsert_is_false
1892
- driver.configure("write_operation update
1893
- id_key request_id")
1894
- stub_elastic_ping
1895
- stub_elastic
1896
- driver.run(default_tag: 'test') do
1897
- driver.feed(sample_record)
1898
- end
1899
- assert(index_cmds[0].has_key?("update"))
1900
- assert(!index_cmds[1]["doc_as_upsert"])
1901
- assert(!index_cmds[1]["upsert"])
1902
- end
1903
-
1904
- def test_update_should_remove_keys_from_doc_when_keys_are_skipped
1905
- driver.configure("write_operation update
1906
- id_key request_id
1907
- remove_keys_on_update parent_id")
1908
- stub_elastic_ping
1909
- stub_elastic
1910
- driver.run(default_tag: 'test') do
1911
- driver.feed(sample_record)
1912
- end
1913
- assert(index_cmds[1]["doc"])
1914
- assert(!index_cmds[1]["doc"]["parent_id"])
1915
- end
1916
-
1917
- def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
1918
- driver.configure("write_operation upsert
1919
- id_key request_id")
1920
- stub_elastic_ping
1921
- stub_elastic
1922
- driver.run(default_tag: 'test') do
1923
- driver.feed(sample_record)
1924
- end
1925
- assert(index_cmds[0].has_key?("update"))
1926
- assert(index_cmds[1]["doc_as_upsert"])
1927
- assert(!index_cmds[1]["upsert"])
1928
- end
1929
-
1930
- def test_upsert_should_write_update_op_upsert_and_doc_when_keys_are_skipped
1931
- driver.configure("write_operation upsert
1932
- id_key request_id
1933
- remove_keys_on_update parent_id")
1934
- stub_elastic_ping
1935
- stub_elastic
1936
- driver.run(default_tag: 'test') do
1937
- driver.feed(sample_record)
1938
- end
1939
- assert(index_cmds[0].has_key?("update"))
1940
- assert(!index_cmds[1]["doc_as_upsert"])
1941
- assert(index_cmds[1]["upsert"])
1942
- assert(index_cmds[1]["doc"])
1943
- end
1944
-
1945
- def test_upsert_should_remove_keys_from_doc_when_keys_are_skipped
1946
- driver.configure("write_operation upsert
1947
- id_key request_id
1948
- remove_keys_on_update parent_id")
1949
- stub_elastic_ping
1950
- stub_elastic
1951
- driver.run(default_tag: 'test') do
1952
- driver.feed(sample_record)
1953
- end
1954
- assert(index_cmds[1]["upsert"] != index_cmds[1]["doc"])
1955
- assert(!index_cmds[1]["doc"]["parent_id"])
1956
- assert(index_cmds[1]["upsert"]["parent_id"])
1957
- end
1958
-
1959
- def test_upsert_should_remove_multiple_keys_when_keys_are_skipped
1960
- driver.configure("write_operation upsert
1961
- id_key id
1962
- remove_keys_on_update foo,baz")
1963
- stub_elastic_ping
1964
- stub_elastic
1965
- driver.run(default_tag: 'test') do
1966
- driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
1967
- end
1968
- assert(
1969
- index_cmds[1]["doc"] == {
1970
- "id" => 1,
1971
- "zip" => "zam",
1972
- }
1973
- )
1974
- assert(
1975
- index_cmds[1]["upsert"] == {
1976
- "id" => 1,
1977
- "foo" => "bar",
1978
- "baz" => "quix",
1979
- "zip" => "zam",
1980
- }
1981
- )
1982
- end
1983
-
1984
- def test_upsert_should_remove_keys_from_when_the_keys_are_in_the_record
1985
- driver.configure("write_operation upsert
1986
- id_key id
1987
- remove_keys_on_update_key keys_to_skip")
1988
- stub_elastic_ping
1989
- stub_elastic
1990
- driver.run(default_tag: 'test') do
1991
- driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
1992
- end
1993
- assert(
1994
- index_cmds[1]["doc"] == {
1995
- "id" => 1,
1996
- "foo" => "bar",
1997
- }
1998
- )
1999
- assert(
2000
- index_cmds[1]["upsert"] == {
2001
- "id" => 1,
2002
- "foo" => "bar",
2003
- "baz" => "quix",
2004
- }
2005
- )
2006
- end
2007
-
2008
- def test_upsert_should_remove_keys_from_key_on_record_has_higher_presedence_than_config
2009
- driver.configure("write_operation upsert
2010
- id_key id
2011
- remove_keys_on_update foo,bar
2012
- remove_keys_on_update_key keys_to_skip")
2013
- stub_elastic_ping
2014
- stub_elastic
2015
- driver.run(default_tag: 'test') do
2016
- driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
2017
- end
2018
- assert(
2019
- index_cmds[1]["doc"] == {
2020
- "id" => 1,
2021
- # we only expect baz to be stripped here, if the config was more important
2022
- # foo would be stripped too.
2023
- "foo" => "bar",
2024
- }
2025
- )
2026
- assert(
2027
- index_cmds[1]["upsert"] == {
2028
- "id" => 1,
2029
- "foo" => "bar",
2030
- "baz" => "quix",
2031
- }
2032
- )
2033
- end
2034
-
2035
- def test_create_should_write_create_op
2036
- driver.configure("write_operation create
2037
- id_key request_id")
2038
- stub_elastic_ping
2039
- stub_elastic
2040
- driver.run(default_tag: 'test') do
2041
- driver.feed(sample_record)
2042
- end
2043
- assert(index_cmds[0].has_key?("create"))
2044
- end
2045
-
2046
- end
1
+ require 'helper'
2
+ require 'date'
3
+ require 'fluent/test/helpers'
4
+ require 'json'
5
+ require 'fluent/test/driver/output'
6
+ require 'flexmock/test_unit'
7
+
8
+ class ElasticsearchOutput < Test::Unit::TestCase
9
+ include FlexMock::TestCase
10
+ include Fluent::Test::Helpers
11
+
12
+ attr_accessor :index_cmds, :index_command_counts
13
+
14
+ def setup
15
+ Fluent::Test.setup
16
+ require 'fluent/plugin/out_elasticsearch'
17
+ @driver = nil
18
+ log = Fluent::Engine.log
19
+ log.out.logs.slice!(0, log.out.logs.length)
20
+ end
21
+
22
+ def driver(conf='', es_version=5)
23
+ # For request stub to detect compatibility.
24
+ @es_version ||= es_version
25
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
26
+ def detect_es_major_version
27
+ #{@es_version}
28
+ end
29
+ CODE
30
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
31
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
32
+ if !defined?(Fluent::Plugin::Output)
33
+ def format(tag, time, record)
34
+ [time, record].to_msgpack
35
+ end
36
+ end
37
+ }.configure(conf)
38
+ end
39
+
40
+ def default_type_name
41
+ Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
42
+ end
43
+
44
+ def sample_record
45
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
46
+ end
47
+
48
+ def nested_sample_record
49
+ {'nested' =>
50
+ {'age' => 26, 'parent_id' => 'parent', 'routing_id' => 'routing', 'request_id' => '42'}
51
+ }
52
+ end
53
+
54
+ def stub_elastic_ping(url="http://localhost:9200")
55
+ stub_request(:head, url).to_return(:status => 200, :body => "", :headers => {})
56
+ end
57
+
58
+ def stub_elastic(url="http://localhost:9200/_bulk")
59
+ stub_request(:post, url).with do |req|
60
+ @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
61
+ end
62
+ end
63
+
64
+ def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
65
+ stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
66
+ end
67
+
68
+ def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
69
+ if @index_command_counts == nil
70
+ @index_command_counts = {}
71
+ @index_command_counts.default = 0
72
+ end
73
+
74
+ stub_request(:post, url).with do |req|
75
+ index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
76
+ @index_command_counts[url] += index_cmds.size
77
+ end
78
+ end
79
+
80
+ def make_response_body(req, error_el = nil, error_status = nil, error = nil)
81
+ req_index_cmds = req.body.split("\n").map { |r| JSON.parse(r) }
82
+ items = []
83
+ count = 0
84
+ ids = 1
85
+ op = nil
86
+ index = nil
87
+ type = nil
88
+ id = nil
89
+ req_index_cmds.each do |cmd|
90
+ if count.even?
91
+ op = cmd.keys[0]
92
+ index = cmd[op]['_index']
93
+ type = cmd[op]['_type']
94
+ if cmd[op].has_key?('_id')
95
+ id = cmd[op]['_id']
96
+ else
97
+ # Note: this appears to be an undocumented feature of Elasticsearch
98
+ # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/docs-bulk.html
99
+ # When you submit an "index" write_operation, with no "_id" field in the
100
+ # metadata header, Elasticsearch will turn this into a "create"
101
+ # operation in the response.
102
+ if "index" == op
103
+ op = "create"
104
+ end
105
+ id = ids
106
+ ids += 1
107
+ end
108
+ else
109
+ item = {
110
+ op => {
111
+ '_index' => index, '_type' => type, '_id' => id, '_version' => 1,
112
+ '_shards' => { 'total' => 1, 'successful' => 1, 'failed' => 0 },
113
+ 'status' => op == 'create' ? 201 : 200
114
+ }
115
+ }
116
+ items.push(item)
117
+ end
118
+ count += 1
119
+ end
120
+ if !error_el.nil? && !error_status.nil? && !error.nil?
121
+ op = items[error_el].keys[0]
122
+ items[error_el][op].delete('_version')
123
+ items[error_el][op].delete('_shards')
124
+ items[error_el][op]['error'] = error
125
+ items[error_el][op]['status'] = error_status
126
+ errors = true
127
+ else
128
+ errors = false
129
+ end
130
+ @index_cmds = items
131
+ body = { 'took' => 6, 'errors' => errors, 'items' => items }
132
+ return body.to_json
133
+ end
134
+
135
+ def stub_elastic_bad_argument(url="http://localhost:9200/_bulk")
136
+ error = {
137
+ "type" => "mapper_parsing_exception",
138
+ "reason" => "failed to parse [...]",
139
+ "caused_by" => {
140
+ "type" => "illegal_argument_exception",
141
+ "reason" => "Invalid format: \"...\""
142
+ }
143
+ }
144
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 400, error), :headers => { 'Content-Type' => 'json' } } })
145
+ end
146
+
147
+ def stub_elastic_bulk_error(url="http://localhost:9200/_bulk")
148
+ error = {
149
+ "type" => "some-unrecognized-error",
150
+ "reason" => "some message printed here ...",
151
+ }
152
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
153
+ end
154
+
155
+ def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
156
+ error = {
157
+ "status" => 500,
158
+ "type" => "es_rejected_execution_exception",
159
+ "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
160
+ }
161
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 429, error), :headers => { 'Content-Type' => 'json' } } })
162
+ end
163
+
164
+ def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
165
+ error = {
166
+ "status" => 500,
167
+ "type" => "out_of_memory_error",
168
+ "reason" => "Java heap space"
169
+ }
170
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
171
+ end
172
+
173
+ def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
174
+ error = {
175
+ "category" => "some-other-type",
176
+ "reason" => "some-other-reason"
177
+ }
178
+ stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
179
+ end
180
+
181
+ def assert_logs_include(logs, msg)
182
+ matches = logs.grep /#{msg}/
183
+ assert_equal(1, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
184
+ end
185
+
186
+ def test_configure
187
+ config = %{
188
+ host logs.google.com
189
+ port 777
190
+ scheme https
191
+ path /es/
192
+ user john
193
+ password doe
194
+ }
195
+ instance = driver(config).instance
196
+
197
+ assert_equal 'logs.google.com', instance.host
198
+ assert_equal 777, instance.port
199
+ assert_equal 'https', instance.scheme
200
+ assert_equal '/es/', instance.path
201
+ assert_equal 'john', instance.user
202
+ assert_equal 'doe', instance.password
203
+ assert_equal :TLSv1, instance.ssl_version
204
+ assert_nil instance.client_key
205
+ assert_nil instance.client_cert
206
+ assert_nil instance.client_key_pass
207
+ assert_false instance.with_transporter_log
208
+ assert_equal :"application/json", instance.content_type
209
+ assert_equal "fluentd", default_type_name
210
+ end
211
+
212
+ test 'configure Content-Type' do
213
+ config = %{
214
+ content_type application/x-ndjson
215
+ }
216
+ instance = driver(config).instance
217
+ assert_equal :"application/x-ndjson", instance.content_type
218
+ end
219
+
220
+ test 'invalid Content-Type' do
221
+ config = %{
222
+ content_type nonexistent/invalid
223
+ }
224
+ assert_raise(Fluent::ConfigError) {
225
+ instance = driver(config).instance
226
+ }
227
+ end
228
+
229
+ test 'invalid specification of times of retrying template installation' do
230
+ config = %{
231
+ max_retry_putting_template -3
232
+ }
233
+ assert_raise(Fluent::ConfigError) {
234
+ instance = driver(config).instance
235
+ }
236
+ end
237
+
238
+ test 'Detected Elasticsearch 7' do
239
+ config = %{
240
+ type_name changed
241
+ }
242
+ instance = driver(config, 7).instance
243
+ assert_equal '_doc', instance.type_name
244
+ end
245
+
246
+ test 'lack of tag in chunk_keys' do
247
+ assert_raise_message(/'tag' in chunk_keys is required./) do
248
+ driver(Fluent::Config::Element.new(
249
+ 'ROOT', '', {
250
+ '@type' => 'elasticsearch',
251
+ 'host' => 'log.google.com',
252
+ 'port' => 777,
253
+ 'scheme' => 'https',
254
+ 'path' => '/es/',
255
+ 'user' => 'john',
256
+ 'pasword' => 'doe',
257
+ }, [
258
+ Fluent::Config::Element.new('buffer', 'mykey', {
259
+ 'chunk_keys' => 'mykey'
260
+ }, [])
261
+ ]
262
+ ))
263
+ end
264
+ end
265
+
266
+ def test_template_already_present
267
+ config = %{
268
+ host logs.google.com
269
+ port 777
270
+ scheme https
271
+ path /es/
272
+ user john
273
+ password doe
274
+ template_name logstash
275
+ template_file /abc123
276
+ }
277
+
278
+ # connection start
279
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
280
+ to_return(:status => 200, :body => "", :headers => {})
281
+ # check if template exists
282
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
283
+ to_return(:status => 200, :body => "", :headers => {})
284
+
285
+ driver(config)
286
+
287
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
288
+ end
289
+
290
+ def test_template_create
291
+ cwd = File.dirname(__FILE__)
292
+ template_file = File.join(cwd, 'test_template.json')
293
+
294
+ config = %{
295
+ host logs.google.com
296
+ port 777
297
+ scheme https
298
+ path /es/
299
+ user john
300
+ password doe
301
+ template_name logstash
302
+ template_file #{template_file}
303
+ }
304
+
305
+ # connection start
306
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
307
+ to_return(:status => 200, :body => "", :headers => {})
308
+ # check if template exists
309
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
310
+ to_return(:status => 404, :body => "", :headers => {})
311
+ # creation
312
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
313
+ to_return(:status => 200, :body => "", :headers => {})
314
+
315
+ driver(config)
316
+
317
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
318
+ end
319
+
320
+ def test_template_overwrite
321
+ cwd = File.dirname(__FILE__)
322
+ template_file = File.join(cwd, 'test_template.json')
323
+
324
+ config = %{
325
+ host logs.google.com
326
+ port 777
327
+ scheme https
328
+ path /es/
329
+ user john
330
+ password doe
331
+ template_name logstash
332
+ template_file #{template_file}
333
+ template_overwrite true
334
+ }
335
+
336
+ # connection start
337
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
338
+ to_return(:status => 200, :body => "", :headers => {})
339
+ # check if template exists
340
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
341
+ to_return(:status => 200, :body => "", :headers => {})
342
+ # creation
343
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
344
+ to_return(:status => 200, :body => "", :headers => {})
345
+
346
+ driver(config)
347
+
348
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
349
+ end
350
+
351
+
352
+ def test_template_create_invalid_filename
353
+ config = %{
354
+ host logs.google.com
355
+ port 777
356
+ scheme https
357
+ path /es/
358
+ user john
359
+ password doe
360
+ template_name logstash
361
+ template_file /abc123
362
+ }
363
+
364
+ # connection start
365
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
366
+ to_return(:status => 200, :body => "", :headers => {})
367
+ # check if template exists
368
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
369
+ to_return(:status => 404, :body => "", :headers => {})
370
+
371
+ assert_raise(RuntimeError) {
372
+ driver(config)
373
+ }
374
+ end
375
+
376
+ def test_template_retry_install
377
+ cwd = File.dirname(__FILE__)
378
+ template_file = File.join(cwd, 'test_template.json')
379
+
380
+ config = %{
381
+ host logs.google.com
382
+ port 778
383
+ scheme https
384
+ path /es/
385
+ user john
386
+ password doe
387
+ template_name logstash
388
+ template_file #{template_file}
389
+ max_retry_putting_template 3
390
+ }
391
+
392
+ connection_resets = 0
393
+ # connection start
394
+ stub_request(:head, "https://john:doe@logs.google.com:778/es//").with do |req|
395
+ connection_resets += 1
396
+ end
397
+ # check if template exists
398
+ stub_request(:get, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
399
+ raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
400
+ end
401
+ # creation
402
+ stub_request(:put, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
403
+ raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
404
+ end
405
+
406
+ assert_raise(Fluent::Plugin::ElasticsearchOutput::ConnectionFailure) do
407
+ driver(config)
408
+ end
409
+
410
+ assert_equal(connection_resets, 4)
411
+ end
412
+
413
+ def test_templates_create
414
+ cwd = File.dirname(__FILE__)
415
+ template_file = File.join(cwd, 'test_template.json')
416
+ config = %{
417
+ host logs.google.com
418
+ port 777
419
+ scheme https
420
+ path /es/
421
+ user john
422
+ password doe
423
+ templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
424
+ }
425
+
426
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
427
+ to_return(:status => 200, :body => "", :headers => {})
428
+ # check if template exists
429
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
430
+ to_return(:status => 404, :body => "", :headers => {})
431
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
432
+ to_return(:status => 404, :body => "", :headers => {})
433
+
434
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
435
+ to_return(:status => 200, :body => "", :headers => {}) #exists
436
+
437
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
438
+ to_return(:status => 200, :body => "", :headers => {})
439
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
440
+ to_return(:status => 200, :body => "", :headers => {})
441
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
442
+ to_return(:status => 200, :body => "", :headers => {})
443
+
444
+ driver(config)
445
+
446
+ assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
447
+ assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
448
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
449
+ end
450
+
451
+ def test_templates_overwrite
452
+ cwd = File.dirname(__FILE__)
453
+ template_file = File.join(cwd, 'test_template.json')
454
+ config = %{
455
+ host logs.google.com
456
+ port 777
457
+ scheme https
458
+ path /es/
459
+ user john
460
+ password doe
461
+ templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
462
+ template_overwrite true
463
+ }
464
+
465
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
466
+ to_return(:status => 200, :body => "", :headers => {})
467
+ # check if template exists
468
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
469
+ to_return(:status => 200, :body => "", :headers => {})
470
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
471
+ to_return(:status => 200, :body => "", :headers => {})
472
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
473
+ to_return(:status => 200, :body => "", :headers => {}) #exists
474
+
475
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
476
+ to_return(:status => 200, :body => "", :headers => {})
477
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
478
+ to_return(:status => 200, :body => "", :headers => {})
479
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
480
+ to_return(:status => 200, :body => "", :headers => {})
481
+
482
+ driver(config)
483
+
484
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
485
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
486
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
487
+ end
488
+
489
+ def test_templates_not_used
490
+ cwd = File.dirname(__FILE__)
491
+ template_file = File.join(cwd, 'test_template.json')
492
+
493
+ config = %{
494
+ host logs.google.com
495
+ port 777
496
+ scheme https
497
+ path /es/
498
+ user john
499
+ password doe
500
+ template_name logstash
501
+ template_file #{template_file}
502
+ templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
503
+ }
504
+ # connection start
505
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
506
+ to_return(:status => 200, :body => "", :headers => {})
507
+ # check if template exists
508
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
509
+ to_return(:status => 404, :body => "", :headers => {})
510
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
511
+ to_return(:status => 404, :body => "", :headers => {})
512
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
513
+ to_return(:status => 404, :body => "", :headers => {})
514
+ #creation
515
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
516
+ to_return(:status => 200, :body => "", :headers => {})
517
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
518
+ to_return(:status => 200, :body => "", :headers => {})
519
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
520
+ to_return(:status => 200, :body => "", :headers => {})
521
+
522
+ driver(config)
523
+
524
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
525
+
526
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1")
527
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
528
+ end
529
+
530
+ def test_templates_can_be_partially_created_if_error_occurs
531
+ cwd = File.dirname(__FILE__)
532
+ template_file = File.join(cwd, 'test_template.json')
533
+ config = %{
534
+ host logs.google.com
535
+ port 777
536
+ scheme https
537
+ path /es/
538
+ user john
539
+ password doe
540
+ templates {"logstash1":"#{template_file}", "logstash2":"/abc" }
541
+ }
542
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
543
+ to_return(:status => 200, :body => "", :headers => {})
544
+ # check if template exists
545
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
546
+ to_return(:status => 404, :body => "", :headers => {})
547
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
548
+ to_return(:status => 404, :body => "", :headers => {})
549
+
550
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
551
+ to_return(:status => 200, :body => "", :headers => {})
552
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
553
+ to_return(:status => 200, :body => "", :headers => {})
554
+
555
+ assert_raise(RuntimeError) {
556
+ driver(config)
557
+ }
558
+
559
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
560
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
561
+ end
562
+
563
+ def test_legacy_hosts_list
564
+ config = %{
565
+ hosts host1:50,host2:100,host3
566
+ scheme https
567
+ path /es/
568
+ port 123
569
+ }
570
+ instance = driver(config).instance
571
+
572
+ assert_equal 3, instance.get_connection_options[:hosts].length
573
+ host1, host2, host3 = instance.get_connection_options[:hosts]
574
+
575
+ assert_equal 'host1', host1[:host]
576
+ assert_equal 50, host1[:port]
577
+ assert_equal 'https', host1[:scheme]
578
+ assert_equal '/es/', host2[:path]
579
+ assert_equal 'host3', host3[:host]
580
+ assert_equal 123, host3[:port]
581
+ assert_equal 'https', host3[:scheme]
582
+ assert_equal '/es/', host3[:path]
583
+ end
584
+
585
+ def test_hosts_list
586
+ config = %{
587
+ hosts https://john:password@host1:443/elastic/,http://host2
588
+ path /default_path
589
+ user default_user
590
+ password default_password
591
+ }
592
+ instance = driver(config).instance
593
+
594
+ assert_equal 2, instance.get_connection_options[:hosts].length
595
+ host1, host2 = instance.get_connection_options[:hosts]
596
+
597
+ assert_equal 'host1', host1[:host]
598
+ assert_equal 443, host1[:port]
599
+ assert_equal 'https', host1[:scheme]
600
+ assert_equal 'john', host1[:user]
601
+ assert_equal 'password', host1[:password]
602
+ assert_equal '/elastic/', host1[:path]
603
+
604
+ assert_equal 'host2', host2[:host]
605
+ assert_equal 'http', host2[:scheme]
606
+ assert_equal 'default_user', host2[:user]
607
+ assert_equal 'default_password', host2[:password]
608
+ assert_equal '/default_path', host2[:path]
609
+ end
610
+
611
+ def test_hosts_list_with_escape_placeholders
612
+ config = %{
613
+ hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
614
+ path /default_path
615
+ user default_user
616
+ password default_password
617
+ }
618
+ instance = driver(config).instance
619
+
620
+ assert_equal 2, instance.get_connection_options[:hosts].length
621
+ host1, host2 = instance.get_connection_options[:hosts]
622
+
623
+ assert_equal 'host1', host1[:host]
624
+ assert_equal 443, host1[:port]
625
+ assert_equal 'https', host1[:scheme]
626
+ assert_equal 'j%2Bhn', host1[:user]
627
+ assert_equal 'passw%40rd', host1[:password]
628
+ assert_equal '/elastic/', host1[:path]
629
+
630
+ assert_equal 'host2', host2[:host]
631
+ assert_equal 'http', host2[:scheme]
632
+ assert_equal 'default_user', host2[:user]
633
+ assert_equal 'default_password', host2[:password]
634
+ assert_equal '/default_path', host2[:path]
635
+ end
636
+
637
+ def test_single_host_params_and_defaults
638
+ config = %{
639
+ host logs.google.com
640
+ user john
641
+ password doe
642
+ }
643
+ instance = driver(config).instance
644
+
645
+ assert_equal 1, instance.get_connection_options[:hosts].length
646
+ host1 = instance.get_connection_options[:hosts][0]
647
+
648
+ assert_equal 'logs.google.com', host1[:host]
649
+ assert_equal 9200, host1[:port]
650
+ assert_equal 'http', host1[:scheme]
651
+ assert_equal 'john', host1[:user]
652
+ assert_equal 'doe', host1[:password]
653
+ assert_equal nil, host1[:path]
654
+ end
655
+
656
+ def test_single_host_params_and_defaults_with_escape_placeholders
657
+ config = %{
658
+ host logs.google.com
659
+ user %{j+hn}
660
+ password %{d@e}
661
+ }
662
+ instance = driver(config).instance
663
+
664
+ assert_equal 1, instance.get_connection_options[:hosts].length
665
+ host1 = instance.get_connection_options[:hosts][0]
666
+
667
+ assert_equal 'logs.google.com', host1[:host]
668
+ assert_equal 9200, host1[:port]
669
+ assert_equal 'http', host1[:scheme]
670
+ assert_equal 'j%2Bhn', host1[:user]
671
+ assert_equal 'd%40e', host1[:password]
672
+ assert_equal nil, host1[:path]
673
+ end
674
+
675
+ def test_content_type_header
676
+ stub_request(:head, "http://localhost:9200/").
677
+ to_return(:status => 200, :body => "", :headers => {})
678
+ if Elasticsearch::VERSION >= "6.0.2"
679
+ elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
680
+ with(headers: { "Content-Type" => "application/x-ndjson" })
681
+ else
682
+ elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
683
+ with(headers: { "Content-Type" => "application/json" })
684
+ end
685
+ driver.run(default_tag: 'test') do
686
+ driver.feed(sample_record)
687
+ end
688
+ assert_requested(elastic_request)
689
+ end
690
+
691
+ def test_write_message_with_bad_chunk
692
+ driver.configure("target_index_key bad_value\n@log_level debug\n")
693
+ stub_elastic_ping
694
+ stub_elastic
695
+ driver.run(default_tag: 'test') do
696
+ driver.feed({'bad_value'=>"\255"})
697
+ end
698
+ error_log = driver.error_events.map {|e| e.last.message }
699
+
700
+ assert_logs_include(error_log, /(input string invalid)|(invalid byte sequence in UTF-8)/)
701
+ end
702
+
703
+ def test_writes_to_default_index
704
+ stub_elastic_ping
705
+ stub_elastic
706
+ driver.run(default_tag: 'test') do
707
+ driver.feed(sample_record)
708
+ end
709
+ assert_equal('fluentd', index_cmds.first['index']['_index'])
710
+ end
711
+
712
+ def test_writes_to_default_type
713
+ stub_elastic_ping
714
+ stub_elastic
715
+ driver.run(default_tag: 'test') do
716
+ driver.feed(sample_record)
717
+ end
718
+ assert_equal(default_type_name, index_cmds.first['index']['_type'])
719
+ end
720
+
721
+ def test_writes_to_speficied_index
722
+ driver.configure("index_name myindex\n")
723
+ stub_elastic_ping
724
+ stub_elastic
725
+ driver.run(default_tag: 'test') do
726
+ driver.feed(sample_record)
727
+ end
728
+ assert_equal('myindex', index_cmds.first['index']['_index'])
729
+ end
730
+
731
+ class IndexNamePlaceholdersTest < self
732
+ def test_writes_to_speficied_index_with_tag_placeholder
733
+ driver.configure("index_name myindex.${tag}\n")
734
+ stub_elastic_ping
735
+ stub_elastic
736
+ driver.run(default_tag: 'test') do
737
+ driver.feed(sample_record)
738
+ end
739
+ assert_equal('myindex.test', index_cmds.first['index']['_index'])
740
+ end
741
+
742
+ def test_writes_to_speficied_index_with_time_placeholder
743
+ driver.configure(Fluent::Config::Element.new(
744
+ 'ROOT', '', {
745
+ '@type' => 'elasticsearch',
746
+ 'index_name' => 'myindex.%Y.%m.%d',
747
+ }, [
748
+ Fluent::Config::Element.new('buffer', 'tag,time', {
749
+ 'chunk_keys' => ['tag', 'time'],
750
+ 'timekey' => 3600,
751
+ }, [])
752
+ ]
753
+ ))
754
+ stub_elastic_ping
755
+ stub_elastic
756
+ time = Time.parse Date.today.iso8601
757
+ driver.run(default_tag: 'test') do
758
+ driver.feed(time.to_i, sample_record)
759
+ end
760
+ assert_equal("myindex.#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['index']['_index'])
761
+ end
762
+
763
+ def test_writes_to_speficied_index_with_custom_key_placeholder
764
+ driver.configure(Fluent::Config::Element.new(
765
+ 'ROOT', '', {
766
+ '@type' => 'elasticsearch',
767
+ 'index_name' => 'myindex.${pipeline_id}',
768
+ }, [
769
+ Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
770
+ ]
771
+ ))
772
+ time = Time.parse Date.today.iso8601
773
+ pipeline_id = "mypipeline"
774
+ logstash_index = "myindex.#{pipeline_id}"
775
+ stub_elastic_ping
776
+ stub_elastic
777
+ driver.run(default_tag: 'test') do
778
+ driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
779
+ end
780
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
781
+ end
782
+ end
783
+
784
+ def test_writes_to_speficied_index_uppercase
785
+ driver.configure("index_name MyIndex\n")
786
+ stub_elastic_ping
787
+ stub_elastic
788
+ driver.run(default_tag: 'test') do
789
+ driver.feed(sample_record)
790
+ end
791
+ # Allthough index_name has upper-case characters,
792
+ # it should be set as lower-case when sent to elasticsearch.
793
+ assert_equal('myindex', index_cmds.first['index']['_index'])
794
+ end
795
+
796
+ def test_writes_to_target_index_key
797
+ driver.configure("target_index_key @target_index\n")
798
+ stub_elastic_ping
799
+ stub_elastic
800
+ record = sample_record.clone
801
+ driver.run(default_tag: 'test') do
802
+ driver.feed(sample_record.merge('@target_index' => 'local-override'))
803
+ end
804
+ assert_equal('local-override', index_cmds.first['index']['_index'])
805
+ assert_nil(index_cmds[1]['@target_index'])
806
+ end
807
+
808
+ def test_writes_to_target_index_key_logstash
809
+ driver.configure("target_index_key @target_index
810
+ logstash_format true")
811
+ time = Time.parse Date.today.iso8601
812
+ stub_elastic_ping
813
+ stub_elastic
814
+ driver.run(default_tag: 'test') do
815
+ driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
816
+ end
817
+ assert_equal('local-override', index_cmds.first['index']['_index'])
818
+ end
819
+
820
+ def test_writes_to_target_index_key_logstash_uppercase
821
+ driver.configure("target_index_key @target_index
822
+ logstash_format true")
823
+ time = Time.parse Date.today.iso8601
824
+ stub_elastic_ping
825
+ stub_elastic
826
+ driver.run(default_tag: 'test') do
827
+ driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
828
+ end
829
+ # Allthough @target_index has upper-case characters,
830
+ # it should be set as lower-case when sent to elasticsearch.
831
+ assert_equal('local-override', index_cmds.first['index']['_index'])
832
+ end
833
+
834
+ def test_writes_to_default_index_with_pipeline
835
+ pipeline = "fluentd"
836
+ driver.configure("pipeline #{pipeline}")
837
+ stub_elastic_ping
838
+ stub_elastic
839
+ driver.run(default_tag: 'test') do
840
+ driver.feed(sample_record)
841
+ end
842
+ assert_equal(pipeline, index_cmds.first['index']['pipeline'])
843
+ end
844
+
845
+ def test_writes_to_target_index_key_fallack
846
+ driver.configure("target_index_key @target_index\n")
847
+ stub_elastic_ping
848
+ stub_elastic
849
+ driver.run(default_tag: 'test') do
850
+ driver.feed(sample_record)
851
+ end
852
+ assert_equal('fluentd', index_cmds.first['index']['_index'])
853
+ end
854
+
855
+ def test_writes_to_target_index_key_fallack_logstash
856
+ driver.configure("target_index_key @target_index\n
857
+ logstash_format true")
858
+ time = Time.parse Date.today.iso8601
859
+ logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
860
+ stub_elastic_ping
861
+ stub_elastic
862
+ driver.run(default_tag: 'test') do
863
+ driver.feed(time.to_i, sample_record)
864
+ end
865
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
866
+ end
867
+
868
+ data("border" => {"es_version" => 6, "_type" => "mytype"},
869
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
870
+ )
871
+ def test_writes_to_speficied_type(data)
872
+ driver('', data["es_version"]).configure("type_name mytype\n")
873
+ stub_elastic_ping
874
+ stub_elastic
875
+ driver.run(default_tag: 'test') do
876
+ driver.feed(sample_record)
877
+ end
878
+ assert_equal(data['_type'], index_cmds.first['index']['_type'])
879
+ end
880
+
881
+ data("border" => {"es_version" => 6, "_type" => "mytype.test"},
882
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
883
+ )
884
+ def test_writes_to_speficied_type_with_placeholders(data)
885
+ driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
886
+ stub_elastic_ping
887
+ stub_elastic
888
+ driver.run(default_tag: 'test') do
889
+ driver.feed(sample_record)
890
+ end
891
+ assert_equal(data['_type'], index_cmds.first['index']['_type'])
892
+ end
893
+
894
+ data("old" => {"es_version" => 2, "_type" => "local-override"},
895
+ "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
896
+ "border" => {"es_version" => 6, "_type" => "fluentd"},
897
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
898
+ )
899
+ def test_writes_to_target_type_key(data)
900
+ driver('', data["es_version"]).configure("target_type_key @target_type\n")
901
+ stub_elastic_ping
902
+ stub_elastic
903
+ record = sample_record.clone
904
+ driver.run(default_tag: 'test') do
905
+ driver.feed(sample_record.merge('@target_type' => 'local-override'))
906
+ end
907
+ assert_equal(data["_type"], index_cmds.first['index']['_type'])
908
+ assert_nil(index_cmds[1]['@target_type'])
909
+ end
910
+
911
+ def test_writes_to_target_type_key_fallack_to_default
912
+ driver.configure("target_type_key @target_type\n")
913
+ stub_elastic_ping
914
+ stub_elastic
915
+ driver.run(default_tag: 'test') do
916
+ driver.feed(sample_record)
917
+ end
918
+ assert_equal(default_type_name, index_cmds.first['index']['_type'])
919
+ end
920
+
921
+ def test_writes_to_target_type_key_fallack_to_type_name
922
+ driver.configure("target_type_key @target_type
923
+ type_name mytype")
924
+ stub_elastic_ping
925
+ stub_elastic
926
+ driver.run(default_tag: 'test') do
927
+ driver.feed(sample_record)
928
+ end
929
+ assert_equal('mytype', index_cmds.first['index']['_type'])
930
+ end
931
+
932
+ data("old" => {"es_version" => 2, "_type" => "local-override"},
933
+ "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
934
+ "border" => {"es_version" => 6, "_type" => "fluentd"},
935
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
936
+ )
937
+ def test_writes_to_target_type_key_nested(data)
938
+ driver('', data["es_version"]).configure("target_type_key kubernetes.labels.log_type\n")
939
+ stub_elastic_ping
940
+ stub_elastic
941
+ driver.run(default_tag: 'test') do
942
+ driver.feed(sample_record.merge('kubernetes' => {
943
+ 'labels' => {
944
+ 'log_type' => 'local-override'
945
+ }
946
+ }))
947
+ end
948
+ assert_equal(data["_type"], index_cmds.first['index']['_type'])
949
+ assert_nil(index_cmds[1]['kubernetes']['labels']['log_type'])
950
+ end
951
+
952
+ def test_writes_to_target_type_key_fallack_to_default_nested
953
+ driver.configure("target_type_key kubernetes.labels.log_type\n")
954
+ stub_elastic_ping
955
+ stub_elastic
956
+ driver.run(default_tag: 'test') do
957
+ driver.feed(sample_record.merge('kubernetes' => {
958
+ 'labels' => {
959
+ 'other_labels' => 'test'
960
+ }
961
+ }))
962
+ end
963
+ assert_equal(default_type_name, index_cmds.first['index']['_type'])
964
+ end
965
+
966
+ def test_writes_to_speficied_host
967
+ driver.configure("host 192.168.33.50\n")
968
+ stub_elastic_ping("http://192.168.33.50:9200")
969
+ elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
970
+ driver.run(default_tag: 'test') do
971
+ driver.feed(sample_record)
972
+ end
973
+ assert_requested(elastic_request)
974
+ end
975
+
976
+ def test_writes_to_speficied_port
977
+ driver.configure("port 9201\n")
978
+ stub_elastic_ping("http://localhost:9201")
979
+ elastic_request = stub_elastic("http://localhost:9201/_bulk")
980
+ driver.run(default_tag: 'test') do
981
+ driver.feed(sample_record)
982
+ end
983
+ assert_requested(elastic_request)
984
+ end
985
+
986
+ def test_writes_to_multi_hosts
987
+ hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
988
+ hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
989
+
990
+ driver.configure("hosts #{hosts_string}")
991
+
992
+ hosts.each do |host_info|
993
+ host, port = host_info
994
+ stub_elastic_ping("http://#{host}:#{port}")
995
+ stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
996
+ end
997
+
998
+ driver.run(default_tag: 'test') do
999
+ 1000.times do
1000
+ driver.feed(sample_record.merge('age'=>rand(100)))
1001
+ end
1002
+ end
1003
+
1004
+ # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
1005
+ # it's Fluentd test driver's constraint
1006
+ # so @index_command_counts.size is always 1
1007
+
1008
+ assert(@index_command_counts.size > 0, "not working with hosts options")
1009
+
1010
+ total = 0
1011
+ @index_command_counts.each do |url, count|
1012
+ total += count
1013
+ end
1014
+ assert_equal(2000, total)
1015
+ end
1016
+
1017
+ def test_nested_record_with_flattening_on
1018
+ driver.configure("flatten_hashes true
1019
+ flatten_hashes_separator |")
1020
+
1021
+ original_hash = {"foo" => {"bar" => "baz"}, "people" => [
1022
+ {"age" => "25", "height" => "1ft"},
1023
+ {"age" => "30", "height" => "2ft"}
1024
+ ]}
1025
+
1026
+ expected_output = {"foo|bar"=>"baz", "people" => [
1027
+ {"age" => "25", "height" => "1ft"},
1028
+ {"age" => "30", "height" => "2ft"}
1029
+ ]}
1030
+
1031
+ stub_elastic_ping
1032
+ stub_elastic
1033
+ driver.run(default_tag: 'test') do
1034
+ driver.feed(original_hash)
1035
+ end
1036
+ assert_equal expected_output, index_cmds[1]
1037
+ end
1038
+
1039
+ def test_nested_record_with_flattening_off
1040
+ # flattening off by default
1041
+
1042
+ original_hash = {"foo" => {"bar" => "baz"}}
1043
+ expected_output = {"foo" => {"bar" => "baz"}}
1044
+
1045
+ stub_elastic_ping
1046
+ stub_elastic
1047
+ driver.run(default_tag: 'test') do
1048
+ driver.feed(original_hash)
1049
+ end
1050
+ assert_equal expected_output, index_cmds[1]
1051
+ end
1052
+
1053
+ def test_makes_bulk_request
1054
+ stub_elastic_ping
1055
+ stub_elastic
1056
+ driver.run(default_tag: 'test') do
1057
+ driver.feed(sample_record)
1058
+ driver.feed(sample_record.merge('age' => 27))
1059
+ end
1060
+ assert_equal(4, index_cmds.count)
1061
+ end
1062
+
1063
+ def test_all_records_are_preserved_in_bulk
1064
+ stub_elastic_ping
1065
+ stub_elastic
1066
+ driver.run(default_tag: 'test') do
1067
+ driver.feed(sample_record)
1068
+ driver.feed(sample_record.merge('age' => 27))
1069
+ end
1070
+ assert_equal(26, index_cmds[1]['age'])
1071
+ assert_equal(27, index_cmds[3]['age'])
1072
+ end
1073
+
1074
+ def test_writes_to_logstash_index
1075
+ driver.configure("logstash_format true\n")
1076
+ #
1077
+ # This is 1 second past midnight in BST, so the UTC index should be the day before
1078
+ dt = DateTime.new(2015, 6, 1, 0, 0, 1, "+01:00")
1079
+ logstash_index = "logstash-2015.05.31"
1080
+ stub_elastic_ping
1081
+ stub_elastic
1082
+ driver.run(default_tag: 'test') do
1083
+ driver.feed(dt.to_time.to_i, sample_record)
1084
+ end
1085
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1086
+ end
1087
+
1088
+ def test_writes_to_logstash_non_utc_index
1089
+ driver.configure("logstash_format true
1090
+ utc_index false")
1091
+ # When using `utc_index false` the index time will be the local day of
1092
+ # ingestion time
1093
+ time = Date.today.to_time
1094
+ index = "logstash-#{time.strftime("%Y.%m.%d")}"
1095
+ stub_elastic_ping
1096
+ stub_elastic
1097
+ driver.run(default_tag: 'test') do
1098
+ driver.feed(time.to_i, sample_record)
1099
+ end
1100
+ assert_equal(index, index_cmds.first['index']['_index'])
1101
+ end
1102
+
1103
+ def test_writes_to_logstash_index_with_specified_prefix
1104
+ driver.configure("logstash_format true
1105
+ logstash_prefix myprefix")
1106
+ time = Time.parse Date.today.iso8601
1107
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1108
+ stub_elastic_ping
1109
+ stub_elastic
1110
+ driver.run(default_tag: 'test') do
1111
+ driver.feed(time.to_i, sample_record)
1112
+ end
1113
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1114
+ end
1115
+
1116
+ def test_writes_to_logstash_index_with_specified_prefix_and_separator
1117
+ separator = '_'
1118
+ driver.configure("logstash_format true
1119
+ logstash_prefix_separator #{separator}
1120
+ logstash_prefix myprefix")
1121
+ time = Time.parse Date.today.iso8601
1122
+ logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
1123
+ stub_elastic_ping
1124
+ stub_elastic
1125
+ driver.run(default_tag: 'test') do
1126
+ driver.feed(time.to_i, sample_record)
1127
+ end
1128
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1129
+ end
1130
+
1131
+ class LogStashPrefixPlaceholdersTest < self
1132
+ def test_writes_to_logstash_index_with_specified_prefix_and_tag_placeholder
1133
+ driver.configure("logstash_format true
1134
+ logstash_prefix myprefix-${tag}")
1135
+ time = Time.parse Date.today.iso8601
1136
+ logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
1137
+ stub_elastic_ping
1138
+ stub_elastic
1139
+ driver.run(default_tag: 'test') do
1140
+ driver.feed(time.to_i, sample_record)
1141
+ end
1142
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1143
+ end
1144
+
1145
+ def test_writes_to_logstash_index_with_specified_prefix_and_time_placeholder
1146
+ driver.configure(Fluent::Config::Element.new(
1147
+ 'ROOT', '', {
1148
+ '@type' => 'elasticsearch',
1149
+ 'logstash_format' => true,
1150
+ 'logstash_prefix' => 'myprefix-%H',
1151
+ }, [
1152
+ Fluent::Config::Element.new('buffer', 'tag,time', {
1153
+ 'chunk_keys' => ['tag', 'time'],
1154
+ 'timekey' => 3600,
1155
+ }, [])
1156
+ ]
1157
+ ))
1158
+ time = Time.parse Date.today.iso8601
1159
+ logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
1160
+ stub_elastic_ping
1161
+ stub_elastic
1162
+ driver.run(default_tag: 'test') do
1163
+ driver.feed(time.to_i, sample_record)
1164
+ end
1165
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1166
+ end
1167
+
1168
+ def test_writes_to_logstash_index_with_specified_prefix_and_custom_key_placeholder
1169
+ driver.configure(Fluent::Config::Element.new(
1170
+ 'ROOT', '', {
1171
+ '@type' => 'elasticsearch',
1172
+ 'logstash_format' => true,
1173
+ 'logstash_prefix' => 'myprefix-${pipeline_id}',
1174
+ }, [
1175
+ Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
1176
+ ]
1177
+ ))
1178
+ time = Time.parse Date.today.iso8601
1179
+ pipeline_id = "mypipeline"
1180
+ logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
1181
+ stub_elastic_ping
1182
+ stub_elastic
1183
+ driver.run(default_tag: 'test') do
1184
+ driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
1185
+ end
1186
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1187
+ end
1188
+ end
1189
+
1190
+ def test_writes_to_logstash_index_with_specified_prefix_uppercase
1191
+ driver.configure("logstash_format true
1192
+ logstash_prefix MyPrefix")
1193
+ time = Time.parse Date.today.iso8601
1194
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1195
+ stub_elastic_ping
1196
+ stub_elastic
1197
+ driver.run(default_tag: 'test') do
1198
+ driver.feed(time.to_i, sample_record)
1199
+ end
1200
+ # Allthough logstash_prefix has upper-case characters,
1201
+ # it should be set as lower-case when sent to elasticsearch.
1202
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1203
+ end
1204
+
1205
+ def test_writes_to_logstash_index_with_specified_dateformat
1206
+ driver.configure("logstash_format true
1207
+ logstash_dateformat %Y.%m")
1208
+ time = Time.parse Date.today.iso8601
1209
+ logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
1210
+ stub_elastic_ping
1211
+ stub_elastic
1212
+ driver.run(default_tag: 'test') do
1213
+ driver.feed(time.to_i, sample_record)
1214
+ end
1215
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1216
+ end
1217
+
1218
+ def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
1219
+ driver.configure("logstash_format true
1220
+ logstash_prefix myprefix
1221
+ logstash_dateformat %Y.%m")
1222
+ time = Time.parse Date.today.iso8601
1223
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
1224
+ stub_elastic_ping
1225
+ stub_elastic
1226
+ driver.run(default_tag: 'test') do
1227
+ driver.feed(time.to_i, sample_record)
1228
+ end
1229
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1230
+ end
1231
+
1232
+ def test_error_if_tag_not_in_chunk_keys
1233
+ assert_raise(Fluent::ConfigError) {
1234
+ config = %{
1235
+ <buffer foo>
1236
+ </buffer>
1237
+ }
1238
+ driver.configure(config)
1239
+ }
1240
+ end
1241
+
1242
+ def test_can_use_custom_chunk_along_with_tag
1243
+ config = %{
1244
+ <buffer tag, foo>
1245
+ </buffer>
1246
+ }
1247
+ driver.configure(config)
1248
+ end
1249
+
1250
+ def test_doesnt_add_logstash_timestamp_by_default
1251
+ stub_elastic_ping
1252
+ stub_elastic
1253
+ driver.run(default_tag: 'test') do
1254
+ driver.feed(sample_record)
1255
+ end
1256
+ assert_nil(index_cmds[1]['@timestamp'])
1257
+ end
1258
+
1259
+ def test_adds_timestamp_when_logstash
1260
+ driver.configure("logstash_format true\n")
1261
+ stub_elastic_ping
1262
+ stub_elastic
1263
+ ts = DateTime.now
1264
+ time = Fluent::EventTime.from_time(ts.to_time)
1265
+ driver.run(default_tag: 'test') do
1266
+ driver.feed(time, sample_record)
1267
+ end
1268
+ assert(index_cmds[1].has_key? '@timestamp')
1269
+ assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1270
+ end
1271
+
1272
+ def test_adds_timestamp_when_include_timestamp
1273
+ driver.configure("include_timestamp true\n")
1274
+ stub_elastic_ping
1275
+ stub_elastic
1276
+ ts = DateTime.now
1277
+ time = Fluent::EventTime.from_time(ts.to_time)
1278
+ driver.run(default_tag: 'test') do
1279
+ driver.feed(time, sample_record)
1280
+ end
1281
+ assert(index_cmds[1].has_key? '@timestamp')
1282
+ assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1283
+ end
1284
+
1285
+ def test_uses_custom_timestamp_when_included_in_record
1286
+ driver.configure("logstash_format true\n")
1287
+ stub_elastic_ping
1288
+ stub_elastic
1289
+ ts = DateTime.new(2001,2,3).iso8601
1290
+ driver.run(default_tag: 'test') do
1291
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1292
+ end
1293
+ assert(index_cmds[1].has_key? '@timestamp')
1294
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1295
+ end
1296
+
1297
+ def test_uses_custom_timestamp_when_included_in_record_without_logstash
1298
+ driver.configure("include_timestamp true\n")
1299
+ stub_elastic_ping
1300
+ stub_elastic
1301
+ ts = DateTime.new(2001,2,3).iso8601
1302
+ driver.run(default_tag: 'test') do
1303
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1304
+ end
1305
+ assert(index_cmds[1].has_key? '@timestamp')
1306
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1307
+ end
1308
+
1309
+ def test_uses_custom_time_key
1310
+ driver.configure("logstash_format true
1311
+ time_key vtm\n")
1312
+ stub_elastic_ping
1313
+ stub_elastic
1314
+ ts = DateTime.new(2001,2,3).iso8601(9)
1315
+ driver.run(default_tag: 'test') do
1316
+ driver.feed(sample_record.merge!('vtm' => ts))
1317
+ end
1318
+ assert(index_cmds[1].has_key? '@timestamp')
1319
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1320
+ end
1321
+
1322
+ def test_uses_custom_time_key_with_format
1323
+ driver.configure("logstash_format true
1324
+ time_key_format %Y-%m-%d %H:%M:%S.%N%z
1325
+ time_key vtm\n")
1326
+ stub_elastic_ping
1327
+ stub_elastic
1328
+ ts = "2001-02-03 13:14:01.673+02:00"
1329
+ driver.run(default_tag: 'test') do
1330
+ driver.feed(sample_record.merge!('vtm' => ts))
1331
+ end
1332
+ assert(index_cmds[1].has_key? '@timestamp')
1333
+ assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1334
+ assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1335
+ end
1336
+
1337
+ def test_uses_custom_time_key_with_format_without_logstash
1338
+ driver.configure("include_timestamp true
1339
+ index_name test
1340
+ time_key_format %Y-%m-%d %H:%M:%S.%N%z
1341
+ time_key vtm\n")
1342
+ stub_elastic_ping
1343
+ stub_elastic
1344
+ ts = "2001-02-03 13:14:01.673+02:00"
1345
+ driver.run(default_tag: 'test') do
1346
+ driver.feed(sample_record.merge!('vtm' => ts))
1347
+ end
1348
+ assert(index_cmds[1].has_key? '@timestamp')
1349
+ assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1350
+ assert_equal("test", index_cmds[0]['index']['_index'])
1351
+ end
1352
+
1353
+ def test_uses_custom_time_key_exclude_timekey
1354
+ driver.configure("logstash_format true
1355
+ time_key vtm
1356
+ time_key_exclude_timestamp true\n")
1357
+ stub_elastic_ping
1358
+ stub_elastic
1359
+ ts = DateTime.new(2001,2,3).iso8601
1360
+ driver.run(default_tag: 'test') do
1361
+ driver.feed(sample_record.merge!('vtm' => ts))
1362
+ end
1363
+ assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be messing')
1364
+ end
1365
+
1366
+ def test_uses_custom_time_key_format
1367
+ driver.configure("logstash_format true
1368
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1369
+ stub_elastic_ping
1370
+ stub_elastic
1371
+ ts = "2001-02-03T13:14:01.673+02:00"
1372
+ driver.run(default_tag: 'test') do
1373
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1374
+ end
1375
+ assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1376
+ assert(index_cmds[1].has_key? '@timestamp')
1377
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1378
+ end
1379
+
1380
+ def test_uses_custom_time_key_format_without_logstash
1381
+ driver.configure("include_timestamp true
1382
+ index_name test
1383
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1384
+ stub_elastic_ping
1385
+ stub_elastic
1386
+ ts = "2001-02-03T13:14:01.673+02:00"
1387
+ driver.run(default_tag: 'test') do
1388
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1389
+ end
1390
+ assert_equal("test", index_cmds[0]['index']['_index'])
1391
+ assert(index_cmds[1].has_key? '@timestamp')
1392
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1393
+ end
1394
+
1395
+ data(:default => nil,
1396
+ :custom_tag => 'es_plugin.output.time.error')
1397
+ def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
1398
+ tag_config = tag_for_error ? "time_parse_error_tag #{tag_for_error}" : ''
1399
+ tag_for_error = 'Fluent::ElasticsearchOutput::TimeParser.error' if tag_for_error.nil?
1400
+ driver.configure("logstash_format true
1401
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n#{tag_config}\n")
1402
+ stub_elastic_ping
1403
+ stub_elastic
1404
+
1405
+ ts = "2001/02/03 13:14:01,673+02:00"
1406
+ index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
1407
+
1408
+ flexmock(driver.instance.router).should_receive(:emit_error_event)
1409
+ .with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once
1410
+ driver.run(default_tag: 'test') do
1411
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1412
+ end
1413
+
1414
+ assert_equal(index, index_cmds[0]['index']['_index'])
1415
+ assert(index_cmds[1].has_key? '@timestamp')
1416
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1417
+ end
1418
+
1419
+
1420
+ def test_uses_custom_time_key_format_obscure_format
1421
+ driver.configure("logstash_format true
1422
+ time_key_format %a %b %d %H:%M:%S %Z %Y\n")
1423
+ stub_elastic_ping
1424
+ stub_elastic
1425
+ ts = "Thu Nov 29 14:33:20 GMT 2001"
1426
+ driver.run(default_tag: 'test') do
1427
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1428
+ end
1429
+ assert_equal("logstash-2001.11.29", index_cmds[0]['index']['_index'])
1430
+ assert(index_cmds[1].has_key? '@timestamp')
1431
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1432
+ end
1433
+
1434
+ def test_uses_nanosecond_precision_by_default
1435
+ driver.configure("logstash_format true\n")
1436
+ stub_elastic_ping
1437
+ stub_elastic
1438
+ time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1439
+ driver.run(default_tag: 'test') do
1440
+ driver.feed(time, sample_record)
1441
+ end
1442
+ assert(index_cmds[1].has_key? '@timestamp')
1443
+ assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(9))
1444
+ end
1445
+
1446
+ def test_uses_subsecond_precision_when_configured
1447
+ driver.configure("logstash_format true
1448
+ time_precision 3\n")
1449
+ stub_elastic_ping
1450
+ stub_elastic
1451
+ time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1452
+ driver.run(default_tag: 'test') do
1453
+ driver.feed(time, sample_record)
1454
+ end
1455
+ assert(index_cmds[1].has_key? '@timestamp')
1456
+ assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(3))
1457
+ end
1458
+
1459
+ def test_doesnt_add_tag_key_by_default
1460
+ stub_elastic_ping
1461
+ stub_elastic
1462
+ driver.run(default_tag: 'test') do
1463
+ driver.feed(sample_record)
1464
+ end
1465
+ assert_nil(index_cmds[1]['tag'])
1466
+ end
1467
+
1468
+ def test_adds_tag_key_when_configured
1469
+ driver.configure("include_tag_key true\n")
1470
+ stub_elastic_ping
1471
+ stub_elastic
1472
+ driver.run(default_tag: 'mytag') do
1473
+ driver.feed(sample_record)
1474
+ end
1475
+ assert(index_cmds[1].has_key?('tag'))
1476
+ assert_equal(index_cmds[1]['tag'], 'mytag')
1477
+ end
1478
+
1479
+ def test_adds_id_key_when_configured
1480
+ driver.configure("id_key request_id\n")
1481
+ stub_elastic_ping
1482
+ stub_elastic
1483
+ driver.run(default_tag: 'test') do
1484
+ driver.feed(sample_record)
1485
+ end
1486
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1487
+ end
1488
+
1489
+ class NestedIdKeyTest < self
1490
+ def test_adds_nested_id_key_with_dot
1491
+ driver.configure("id_key nested.request_id\n")
1492
+ stub_elastic_ping
1493
+ stub_elastic
1494
+ driver.run(default_tag: 'test') do
1495
+ driver.feed(nested_sample_record)
1496
+ end
1497
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1498
+ end
1499
+
1500
+ def test_adds_nested_id_key_with_dollar_dot
1501
+ driver.configure("id_key $.nested.request_id\n")
1502
+ stub_elastic_ping
1503
+ stub_elastic
1504
+ driver.run(default_tag: 'test') do
1505
+ driver.feed(nested_sample_record)
1506
+ end
1507
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1508
+ end
1509
+
1510
+ def test_adds_nested_id_key_with_bracket
1511
+ driver.configure("id_key $['nested']['request_id']\n")
1512
+ stub_elastic_ping
1513
+ stub_elastic
1514
+ driver.run(default_tag: 'test') do
1515
+ driver.feed(nested_sample_record)
1516
+ end
1517
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1518
+ end
1519
+ end
1520
+
1521
+ def test_doesnt_add_id_key_if_missing_when_configured
1522
+ driver.configure("id_key another_request_id\n")
1523
+ stub_elastic_ping
1524
+ stub_elastic
1525
+ driver.run(default_tag: 'test') do
1526
+ driver.feed(sample_record)
1527
+ end
1528
+ assert(!index_cmds[0]['index'].has_key?('_id'))
1529
+ end
1530
+
1531
+ def test_adds_id_key_when_not_configured
1532
+ stub_elastic_ping
1533
+ stub_elastic
1534
+ driver.run(default_tag: 'test') do
1535
+ driver.feed(sample_record)
1536
+ end
1537
+ assert(!index_cmds[0]['index'].has_key?('_id'))
1538
+ end
1539
+
1540
+ def test_adds_parent_key_when_configured
1541
+ driver.configure("parent_key parent_id\n")
1542
+ stub_elastic_ping
1543
+ stub_elastic
1544
+ driver.run(default_tag: 'test') do
1545
+ driver.feed(sample_record)
1546
+ end
1547
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1548
+ end
1549
+
1550
+ class NestedParentKeyTest < self
1551
+ def test_adds_nested_parent_key_with_dot
1552
+ driver.configure("parent_key nested.parent_id\n")
1553
+ stub_elastic_ping
1554
+ stub_elastic
1555
+ driver.run(default_tag: 'test') do
1556
+ driver.feed(nested_sample_record)
1557
+ end
1558
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1559
+ end
1560
+
1561
+ def test_adds_nested_parent_key_with_dollar_dot
1562
+ driver.configure("parent_key $.nested.parent_id\n")
1563
+ stub_elastic_ping
1564
+ stub_elastic
1565
+ driver.run(default_tag: 'test') do
1566
+ driver.feed(nested_sample_record)
1567
+ end
1568
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1569
+ end
1570
+
1571
+ def test_adds_nested_parent_key_with_bracket
1572
+ driver.configure("parent_key $['nested']['parent_id']\n")
1573
+ stub_elastic_ping
1574
+ stub_elastic
1575
+ driver.run(default_tag: 'test') do
1576
+ driver.feed(nested_sample_record)
1577
+ end
1578
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1579
+ end
1580
+ end
1581
+
1582
+ def test_doesnt_add_parent_key_if_missing_when_configured
1583
+ driver.configure("parent_key another_parent_id\n")
1584
+ stub_elastic_ping
1585
+ stub_elastic
1586
+ driver.run(default_tag: 'test') do
1587
+ driver.feed(sample_record)
1588
+ end
1589
+ assert(!index_cmds[0]['index'].has_key?('_parent'))
1590
+ end
1591
+
1592
+ def test_adds_parent_key_when_not_configured
1593
+ stub_elastic_ping
1594
+ stub_elastic
1595
+ driver.run(default_tag: 'test') do
1596
+ driver.feed(sample_record)
1597
+ end
1598
+ assert(!index_cmds[0]['index'].has_key?('_parent'))
1599
+ end
1600
+
1601
+ def test_adds_routing_key_when_configured
1602
+ driver.configure("routing_key routing_id\n")
1603
+ stub_elastic_ping
1604
+ stub_elastic
1605
+ driver.run(default_tag: 'test') do
1606
+ driver.feed(sample_record)
1607
+ end
1608
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1609
+ end
1610
+
1611
+ class NestedRoutingKeyTest < self
1612
+ def test_adds_nested_routing_key_with_dot
1613
+ driver.configure("routing_key nested.routing_id\n")
1614
+ stub_elastic_ping
1615
+ stub_elastic
1616
+ driver.run(default_tag: 'test') do
1617
+ driver.feed(nested_sample_record)
1618
+ end
1619
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1620
+ end
1621
+
1622
+ def test_adds_nested_routing_key_with_dollar_dot
1623
+ driver.configure("routing_key $.nested.routing_id\n")
1624
+ stub_elastic_ping
1625
+ stub_elastic
1626
+ driver.run(default_tag: 'test') do
1627
+ driver.feed(nested_sample_record)
1628
+ end
1629
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1630
+ end
1631
+
1632
+ def test_adds_nested_routing_key_with_bracket
1633
+ driver.configure("routing_key $['nested']['routing_id']\n")
1634
+ stub_elastic_ping
1635
+ stub_elastic
1636
+ driver.run(default_tag: 'test') do
1637
+ driver.feed(nested_sample_record)
1638
+ end
1639
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1640
+ end
1641
+ end
1642
+
1643
+ def test_doesnt_add_routing_key_if_missing_when_configured
1644
+ driver.configure("routing_key another_routing_id\n")
1645
+ stub_elastic_ping
1646
+ stub_elastic
1647
+ driver.run(default_tag: 'test') do
1648
+ driver.feed(sample_record)
1649
+ end
1650
+ assert(!index_cmds[0]['index'].has_key?('_routing'))
1651
+ end
1652
+
1653
+ def test_adds_routing_key_when_not_configured
1654
+ stub_elastic_ping
1655
+ stub_elastic
1656
+ driver.run(default_tag: 'test') do
1657
+ driver.feed(sample_record)
1658
+ end
1659
+ assert(!index_cmds[0]['index'].has_key?('_routing'))
1660
+ end
1661
+
1662
+ def test_remove_one_key
1663
+ driver.configure("remove_keys key1\n")
1664
+ stub_elastic_ping
1665
+ stub_elastic
1666
+ driver.run(default_tag: 'test') do
1667
+ driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1668
+ end
1669
+ assert(!index_cmds[1].has_key?('key1'))
1670
+ assert(index_cmds[1].has_key?('key2'))
1671
+ end
1672
+
1673
+ def test_remove_multi_keys
1674
+ driver.configure("remove_keys key1, key2\n")
1675
+ stub_elastic_ping
1676
+ stub_elastic
1677
+ driver.run(default_tag: 'test') do
1678
+ driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1679
+ end
1680
+ assert(!index_cmds[1].has_key?('key1'))
1681
+ assert(!index_cmds[1].has_key?('key2'))
1682
+ end
1683
+
1684
+ def test_request_error
1685
+ stub_elastic_ping
1686
+ stub_elastic_unavailable
1687
+ assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
1688
+ driver.run(default_tag: 'test') do
1689
+ driver.feed(sample_record)
1690
+ end
1691
+ }
1692
+ end
1693
+
1694
+ def test_connection_failed_retry
1695
+ connection_resets = 0
1696
+
1697
+ stub_elastic_ping(url="http://localhost:9200").with do |req|
1698
+ connection_resets += 1
1699
+ end
1700
+
1701
+ stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1702
+ raise Faraday::ConnectionFailed, "Test message"
1703
+ end
1704
+
1705
+ driver.run(default_tag: 'test') do
1706
+ driver.feed(sample_record)
1707
+ end
1708
+ assert_equal(connection_resets, 3)
1709
+ end
1710
+
1711
+ def test_reconnect_on_error_enabled
1712
+ connection_resets = 0
1713
+
1714
+ stub_elastic_ping(url="http://localhost:9200").with do |req|
1715
+ connection_resets += 1
1716
+ end
1717
+
1718
+ stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1719
+ raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1720
+ end
1721
+
1722
+ driver.configure("reconnect_on_error true\n")
1723
+
1724
+ assert_raise(ZeroDivisionError) {
1725
+ driver.run(default_tag: 'test', shutdown: false) do
1726
+ driver.feed(sample_record)
1727
+ end
1728
+ }
1729
+
1730
+ assert_raise(Timeout::Error) {
1731
+ driver.run(default_tag: 'test', shutdown: false) do
1732
+ driver.feed(sample_record)
1733
+ end
1734
+ }
1735
+ # FIXME: Consider keywords arguments in #run and how to test this later.
1736
+ # Because v0.14 test driver does not have 1 to 1 correspondence between #run and #flush in tests.
1737
+ assert_equal(connection_resets, 1)
1738
+ end
1739
+
1740
+ def test_reconnect_on_error_disabled
1741
+ connection_resets = 0
1742
+
1743
+ stub_elastic_ping(url="http://localhost:9200").with do |req|
1744
+ connection_resets += 1
1745
+ end
1746
+
1747
+ stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1748
+ raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1749
+ end
1750
+
1751
+ driver.configure("reconnect_on_error false\n")
1752
+
1753
+ assert_raise(ZeroDivisionError) {
1754
+ driver.run(default_tag: 'test', shutdown: false) do
1755
+ driver.feed(sample_record)
1756
+ end
1757
+ }
1758
+
1759
+ assert_raise(Timeout::Error) {
1760
+ driver.run(default_tag: 'test', shutdown: false) do
1761
+ driver.feed(sample_record)
1762
+ end
1763
+ }
1764
+ assert_equal(connection_resets, 1)
1765
+ end
1766
+
1767
+ def test_bulk_error_retags_when_configured
1768
+ driver.configure("retry_tag retry\n")
1769
+ stub_elastic_ping
1770
+ stub_request(:post, 'http://localhost:9200/_bulk')
1771
+ .to_return(lambda do |req|
1772
+ { :status => 200,
1773
+ :headers => { 'Content-Type' => 'json' },
1774
+ :body => %({
1775
+ "took" : 1,
1776
+ "errors" : true,
1777
+ "items" : [
1778
+ {
1779
+ "create" : {
1780
+ "_index" : "foo",
1781
+ "_type" : "bar",
1782
+ "_id" : "abc",
1783
+ "status" : 500,
1784
+ "error" : {
1785
+ "type" : "some unrecognized type",
1786
+ "reason":"some error to cause version mismatch"
1787
+ }
1788
+ }
1789
+ }
1790
+ ]
1791
+ })
1792
+ }
1793
+ end)
1794
+
1795
+ driver.run(default_tag: 'test') do
1796
+ driver.feed(1, sample_record)
1797
+ end
1798
+
1799
+ assert_equal [['retry', 1, sample_record]], driver.events
1800
+ end
1801
+
1802
+ def test_bulk_error
1803
+ stub_elastic_ping
1804
+ stub_request(:post, 'http://localhost:9200/_bulk')
1805
+ .to_return(lambda do |req|
1806
+ { :status => 200,
1807
+ :headers => { 'Content-Type' => 'json' },
1808
+ :body => %({
1809
+ "took" : 1,
1810
+ "errors" : true,
1811
+ "items" : [
1812
+ {
1813
+ "create" : {
1814
+ "_index" : "foo",
1815
+ "_type" : "bar",
1816
+ "_id" : "abc",
1817
+ "status" : 500,
1818
+ "error" : {
1819
+ "type" : "some unrecognized type",
1820
+ "reason":"some error to cause version mismatch"
1821
+ }
1822
+ }
1823
+ },
1824
+ {
1825
+ "create" : {
1826
+ "_index" : "foo",
1827
+ "_type" : "bar",
1828
+ "_id" : "abc",
1829
+ "status" : 201
1830
+ }
1831
+ },
1832
+ {
1833
+ "create" : {
1834
+ "_index" : "foo",
1835
+ "_type" : "bar",
1836
+ "_id" : "abc",
1837
+ "status" : 500,
1838
+ "error" : {
1839
+ "type" : "some unrecognized type",
1840
+ "reason":"some error to cause version mismatch"
1841
+ }
1842
+ }
1843
+ },
1844
+ {
1845
+ "create" : {
1846
+ "_index" : "foo",
1847
+ "_type" : "bar",
1848
+ "_id" : "abc",
1849
+ "_id" : "abc",
1850
+ "status" : 409
1851
+ }
1852
+ }
1853
+ ]
1854
+ })
1855
+ }
1856
+ end)
1857
+
1858
+ driver.run(default_tag: 'test') do
1859
+ driver.feed(1, sample_record)
1860
+ driver.feed(2, sample_record)
1861
+ driver.feed(3, sample_record)
1862
+ driver.feed(4, sample_record)
1863
+ end
1864
+
1865
+ expect = [['test', 1, sample_record],
1866
+ ['test', 3, sample_record]]
1867
+ assert_equal expect, driver.events
1868
+ end
1869
+
1870
+ def test_update_should_not_write_if_theres_no_id
1871
+ driver.configure("write_operation update\n")
1872
+ stub_elastic_ping
1873
+ stub_elastic
1874
+ driver.run(default_tag: 'test') do
1875
+ driver.feed(sample_record)
1876
+ end
1877
+ assert_nil(index_cmds)
1878
+ end
1879
+
1880
+ def test_upsert_should_not_write_if_theres_no_id
1881
+ driver.configure("write_operation upsert\n")
1882
+ stub_elastic_ping
1883
+ stub_elastic
1884
+ driver.run(default_tag: 'test') do
1885
+ driver.feed(sample_record)
1886
+ end
1887
+ assert_nil(index_cmds)
1888
+ end
1889
+
1890
+ def test_create_should_not_write_if_theres_no_id
1891
+ driver.configure("write_operation create\n")
1892
+ stub_elastic_ping
1893
+ stub_elastic
1894
+ driver.run(default_tag: 'test') do
1895
+ driver.feed(sample_record)
1896
+ end
1897
+ assert_nil(index_cmds)
1898
+ end
1899
+
1900
+ def test_update_should_write_update_op_and_doc_as_upsert_is_false
1901
+ driver.configure("write_operation update
1902
+ id_key request_id")
1903
+ stub_elastic_ping
1904
+ stub_elastic
1905
+ driver.run(default_tag: 'test') do
1906
+ driver.feed(sample_record)
1907
+ end
1908
+ assert(index_cmds[0].has_key?("update"))
1909
+ assert(!index_cmds[1]["doc_as_upsert"])
1910
+ assert(!index_cmds[1]["upsert"])
1911
+ end
1912
+
1913
+ def test_update_should_remove_keys_from_doc_when_keys_are_skipped
1914
+ driver.configure("write_operation update
1915
+ id_key request_id
1916
+ remove_keys_on_update parent_id")
1917
+ stub_elastic_ping
1918
+ stub_elastic
1919
+ driver.run(default_tag: 'test') do
1920
+ driver.feed(sample_record)
1921
+ end
1922
+ assert(index_cmds[1]["doc"])
1923
+ assert(!index_cmds[1]["doc"]["parent_id"])
1924
+ end
1925
+
1926
+ def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
1927
+ driver.configure("write_operation upsert
1928
+ id_key request_id")
1929
+ stub_elastic_ping
1930
+ stub_elastic
1931
+ driver.run(default_tag: 'test') do
1932
+ driver.feed(sample_record)
1933
+ end
1934
+ assert(index_cmds[0].has_key?("update"))
1935
+ assert(index_cmds[1]["doc_as_upsert"])
1936
+ assert(!index_cmds[1]["upsert"])
1937
+ end
1938
+
1939
+ def test_upsert_should_write_update_op_upsert_and_doc_when_keys_are_skipped
1940
+ driver.configure("write_operation upsert
1941
+ id_key request_id
1942
+ remove_keys_on_update parent_id")
1943
+ stub_elastic_ping
1944
+ stub_elastic
1945
+ driver.run(default_tag: 'test') do
1946
+ driver.feed(sample_record)
1947
+ end
1948
+ assert(index_cmds[0].has_key?("update"))
1949
+ assert(!index_cmds[1]["doc_as_upsert"])
1950
+ assert(index_cmds[1]["upsert"])
1951
+ assert(index_cmds[1]["doc"])
1952
+ end
1953
+
1954
+ def test_upsert_should_remove_keys_from_doc_when_keys_are_skipped
1955
+ driver.configure("write_operation upsert
1956
+ id_key request_id
1957
+ remove_keys_on_update parent_id")
1958
+ stub_elastic_ping
1959
+ stub_elastic
1960
+ driver.run(default_tag: 'test') do
1961
+ driver.feed(sample_record)
1962
+ end
1963
+ assert(index_cmds[1]["upsert"] != index_cmds[1]["doc"])
1964
+ assert(!index_cmds[1]["doc"]["parent_id"])
1965
+ assert(index_cmds[1]["upsert"]["parent_id"])
1966
+ end
1967
+
1968
+ def test_upsert_should_remove_multiple_keys_when_keys_are_skipped
1969
+ driver.configure("write_operation upsert
1970
+ id_key id
1971
+ remove_keys_on_update foo,baz")
1972
+ stub_elastic_ping
1973
+ stub_elastic
1974
+ driver.run(default_tag: 'test') do
1975
+ driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
1976
+ end
1977
+ assert(
1978
+ index_cmds[1]["doc"] == {
1979
+ "id" => 1,
1980
+ "zip" => "zam",
1981
+ }
1982
+ )
1983
+ assert(
1984
+ index_cmds[1]["upsert"] == {
1985
+ "id" => 1,
1986
+ "foo" => "bar",
1987
+ "baz" => "quix",
1988
+ "zip" => "zam",
1989
+ }
1990
+ )
1991
+ end
1992
+
1993
+ def test_upsert_should_remove_keys_from_when_the_keys_are_in_the_record
1994
+ driver.configure("write_operation upsert
1995
+ id_key id
1996
+ remove_keys_on_update_key keys_to_skip")
1997
+ stub_elastic_ping
1998
+ stub_elastic
1999
+ driver.run(default_tag: 'test') do
2000
+ driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
2001
+ end
2002
+ assert(
2003
+ index_cmds[1]["doc"] == {
2004
+ "id" => 1,
2005
+ "foo" => "bar",
2006
+ }
2007
+ )
2008
+ assert(
2009
+ index_cmds[1]["upsert"] == {
2010
+ "id" => 1,
2011
+ "foo" => "bar",
2012
+ "baz" => "quix",
2013
+ }
2014
+ )
2015
+ end
2016
+
2017
+ def test_upsert_should_remove_keys_from_key_on_record_has_higher_presedence_than_config
2018
+ driver.configure("write_operation upsert
2019
+ id_key id
2020
+ remove_keys_on_update foo,bar
2021
+ remove_keys_on_update_key keys_to_skip")
2022
+ stub_elastic_ping
2023
+ stub_elastic
2024
+ driver.run(default_tag: 'test') do
2025
+ driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
2026
+ end
2027
+ assert(
2028
+ index_cmds[1]["doc"] == {
2029
+ "id" => 1,
2030
+ # we only expect baz to be stripped here, if the config was more important
2031
+ # foo would be stripped too.
2032
+ "foo" => "bar",
2033
+ }
2034
+ )
2035
+ assert(
2036
+ index_cmds[1]["upsert"] == {
2037
+ "id" => 1,
2038
+ "foo" => "bar",
2039
+ "baz" => "quix",
2040
+ }
2041
+ )
2042
+ end
2043
+
2044
+ def test_create_should_write_create_op
2045
+ driver.configure("write_operation create
2046
+ id_key request_id")
2047
+ stub_elastic_ping
2048
+ stub_elastic
2049
+ driver.run(default_tag: 'test') do
2050
+ driver.feed(sample_record)
2051
+ end
2052
+ assert(index_cmds[0].has_key?("create"))
2053
+ end
2054
+
2055
+ end