fluent-plugin-elasticsearch 2.10.2 → 2.10.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (27) hide show
  1. checksums.yaml +4 -4
  2. data/.coveralls.yml +2 -2
  3. data/.editorconfig +9 -9
  4. data/.gitignore +18 -18
  5. data/.travis.yml +14 -14
  6. data/Gemfile +8 -8
  7. data/History.md +239 -236
  8. data/ISSUE_TEMPLATE.md +25 -25
  9. data/LICENSE.txt +201 -201
  10. data/PULL_REQUEST_TEMPLATE.md +10 -10
  11. data/README.md +807 -795
  12. data/Rakefile +11 -11
  13. data/appveyor.yml +35 -35
  14. data/fluent-plugin-elasticsearch.gemspec +31 -31
  15. data/lib/fluent/plugin/elasticsearch_constants.rb +13 -13
  16. data/lib/fluent/plugin/elasticsearch_error_handler.rb +87 -87
  17. data/lib/fluent/plugin/elasticsearch_index_template.rb +60 -42
  18. data/lib/fluent/plugin/filter_elasticsearch_genid.rb +25 -25
  19. data/lib/fluent/plugin/out_elasticsearch.rb +538 -533
  20. data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +294 -294
  21. data/test/helper.rb +24 -24
  22. data/test/plugin/test_elasticsearch_error_handler.rb +195 -195
  23. data/test/plugin/test_filter_elasticsearch_genid.rb +44 -44
  24. data/test/plugin/test_out_elasticsearch.rb +2046 -2009
  25. data/test/plugin/test_out_elasticsearch_dynamic.rb +992 -992
  26. data/test/plugin/test_template.json +23 -23
  27. metadata +3 -3
data/test/plugin/test_out_elasticsearch.rb CHANGED
@@ -1,2009 +1,2046 @@
1
- require 'helper'
2
- require 'date'
3
- require 'fluent/test/helpers'
4
- require 'json'
5
- require 'fluent/test/driver/output'
6
- require 'flexmock/test_unit'
7
-
8
- class ElasticsearchOutput < Test::Unit::TestCase
9
- include FlexMock::TestCase
10
- include Fluent::Test::Helpers
11
-
12
- attr_accessor :index_cmds, :index_command_counts
13
-
14
- def setup
15
- Fluent::Test.setup
16
- require 'fluent/plugin/out_elasticsearch'
17
- @driver = nil
18
- log = Fluent::Engine.log
19
- log.out.logs.slice!(0, log.out.logs.length)
20
- end
21
-
22
- def driver(conf='', es_version=5)
23
- # For request stub to detect compatibility.
24
- @es_version ||= es_version
25
- Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
26
- def detect_es_major_version
27
- #{@es_version}
28
- end
29
- CODE
30
- @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
31
- # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
32
- if !defined?(Fluent::Plugin::Output)
33
- def format(tag, time, record)
34
- [time, record].to_msgpack
35
- end
36
- end
37
- }.configure(conf)
38
- end
39
-
40
- def default_type_name
41
- Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
42
- end
43
-
44
- def sample_record
45
- {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
46
- end
47
-
48
- def nested_sample_record
49
- {'nested' =>
50
- {'age' => 26, 'parent_id' => 'parent', 'routing_id' => 'routing', 'request_id' => '42'}
51
- }
52
- end
53
-
54
- def stub_elastic_ping(url="http://localhost:9200")
55
- stub_request(:head, url).to_return(:status => 200, :body => "", :headers => {})
56
- end
57
-
58
- def stub_elastic(url="http://localhost:9200/_bulk")
59
- stub_request(:post, url).with do |req|
60
- @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
61
- end
62
- end
63
-
64
- def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
65
- stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
66
- end
67
-
68
- def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
69
- if @index_command_counts == nil
70
- @index_command_counts = {}
71
- @index_command_counts.default = 0
72
- end
73
-
74
- stub_request(:post, url).with do |req|
75
- index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
76
- @index_command_counts[url] += index_cmds.size
77
- end
78
- end
79
-
80
- def make_response_body(req, error_el = nil, error_status = nil, error = nil)
81
- req_index_cmds = req.body.split("\n").map { |r| JSON.parse(r) }
82
- items = []
83
- count = 0
84
- ids = 1
85
- op = nil
86
- index = nil
87
- type = nil
88
- id = nil
89
- req_index_cmds.each do |cmd|
90
- if count.even?
91
- op = cmd.keys[0]
92
- index = cmd[op]['_index']
93
- type = cmd[op]['_type']
94
- if cmd[op].has_key?('_id')
95
- id = cmd[op]['_id']
96
- else
97
- # Note: this appears to be an undocumented feature of Elasticsearch
98
- # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/docs-bulk.html
99
- # When you submit an "index" write_operation, with no "_id" field in the
100
- # metadata header, Elasticsearch will turn this into a "create"
101
- # operation in the response.
102
- if "index" == op
103
- op = "create"
104
- end
105
- id = ids
106
- ids += 1
107
- end
108
- else
109
- item = {
110
- op => {
111
- '_index' => index, '_type' => type, '_id' => id, '_version' => 1,
112
- '_shards' => { 'total' => 1, 'successful' => 1, 'failed' => 0 },
113
- 'status' => op == 'create' ? 201 : 200
114
- }
115
- }
116
- items.push(item)
117
- end
118
- count += 1
119
- end
120
- if !error_el.nil? && !error_status.nil? && !error.nil?
121
- op = items[error_el].keys[0]
122
- items[error_el][op].delete('_version')
123
- items[error_el][op].delete('_shards')
124
- items[error_el][op]['error'] = error
125
- items[error_el][op]['status'] = error_status
126
- errors = true
127
- else
128
- errors = false
129
- end
130
- @index_cmds = items
131
- body = { 'took' => 6, 'errors' => errors, 'items' => items }
132
- return body.to_json
133
- end
134
-
135
- def stub_elastic_bad_argument(url="http://localhost:9200/_bulk")
136
- error = {
137
- "type" => "mapper_parsing_exception",
138
- "reason" => "failed to parse [...]",
139
- "caused_by" => {
140
- "type" => "illegal_argument_exception",
141
- "reason" => "Invalid format: \"...\""
142
- }
143
- }
144
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 400, error), :headers => { 'Content-Type' => 'json' } } })
145
- end
146
-
147
- def stub_elastic_bulk_error(url="http://localhost:9200/_bulk")
148
- error = {
149
- "type" => "some-unrecognized-error",
150
- "reason" => "some message printed here ...",
151
- }
152
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
153
- end
154
-
155
- def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
156
- error = {
157
- "status" => 500,
158
- "type" => "es_rejected_execution_exception",
159
- "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
160
- }
161
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 429, error), :headers => { 'Content-Type' => 'json' } } })
162
- end
163
-
164
- def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
165
- error = {
166
- "status" => 500,
167
- "type" => "out_of_memory_error",
168
- "reason" => "Java heap space"
169
- }
170
- stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
171
- end
172
-
173
- def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
174
- error = {
175
- "category" => "some-other-type",
176
- "reason" => "some-other-reason"
177
- }
178
- stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
179
- end
180
-
181
- def assert_logs_include(logs, msg)
182
- matches = logs.grep /#{msg}/
183
- assert_equal(1, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
184
- end
185
-
186
- def test_configure
187
- config = %{
188
- host logs.google.com
189
- port 777
190
- scheme https
191
- path /es/
192
- user john
193
- password doe
194
- }
195
- instance = driver(config).instance
196
-
197
- assert_equal 'logs.google.com', instance.host
198
- assert_equal 777, instance.port
199
- assert_equal 'https', instance.scheme
200
- assert_equal '/es/', instance.path
201
- assert_equal 'john', instance.user
202
- assert_equal 'doe', instance.password
203
- assert_equal :TLSv1, instance.ssl_version
204
- assert_nil instance.client_key
205
- assert_nil instance.client_cert
206
- assert_nil instance.client_key_pass
207
- assert_false instance.with_transporter_log
208
- assert_equal :"application/json", instance.content_type
209
- assert_equal "fluentd", default_type_name
210
- end
211
-
212
- test 'configure Content-Type' do
213
- config = %{
214
- content_type application/x-ndjson
215
- }
216
- instance = driver(config).instance
217
- assert_equal :"application/x-ndjson", instance.content_type
218
- end
219
-
220
- test 'invalid Content-Type' do
221
- config = %{
222
- content_type nonexistent/invalid
223
- }
224
- assert_raise(Fluent::ConfigError) {
225
- instance = driver(config).instance
226
- }
227
- end
228
-
229
- test 'Detected Elasticsearch 7' do
230
- config = %{
231
- type_name changed
232
- }
233
- instance = driver(config, 7).instance
234
- assert_equal '_doc', instance.type_name
235
- end
236
-
237
- test 'lack of tag in chunk_keys' do
238
- assert_raise_message(/'tag' in chunk_keys is required./) do
239
- driver(Fluent::Config::Element.new(
240
- 'ROOT', '', {
241
- '@type' => 'elasticsearch',
242
- 'host' => 'log.google.com',
243
- 'port' => 777,
244
- 'scheme' => 'https',
245
- 'path' => '/es/',
246
- 'user' => 'john',
247
- 'pasword' => 'doe',
248
- }, [
249
- Fluent::Config::Element.new('buffer', 'mykey', {
250
- 'chunk_keys' => 'mykey'
251
- }, [])
252
- ]
253
- ))
254
- end
255
- end
256
-
257
- def test_template_already_present
258
- config = %{
259
- host logs.google.com
260
- port 777
261
- scheme https
262
- path /es/
263
- user john
264
- password doe
265
- template_name logstash
266
- template_file /abc123
267
- }
268
-
269
- # connection start
270
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
271
- to_return(:status => 200, :body => "", :headers => {})
272
- # check if template exists
273
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
274
- to_return(:status => 200, :body => "", :headers => {})
275
-
276
- driver(config)
277
-
278
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
279
- end
280
-
281
- def test_template_create
282
- cwd = File.dirname(__FILE__)
283
- template_file = File.join(cwd, 'test_template.json')
284
-
285
- config = %{
286
- host logs.google.com
287
- port 777
288
- scheme https
289
- path /es/
290
- user john
291
- password doe
292
- template_name logstash
293
- template_file #{template_file}
294
- }
295
-
296
- # connection start
297
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
298
- to_return(:status => 200, :body => "", :headers => {})
299
- # check if template exists
300
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
301
- to_return(:status => 404, :body => "", :headers => {})
302
- # creation
303
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
304
- to_return(:status => 200, :body => "", :headers => {})
305
-
306
- driver(config)
307
-
308
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
309
- end
310
-
311
- def test_template_overwrite
312
- cwd = File.dirname(__FILE__)
313
- template_file = File.join(cwd, 'test_template.json')
314
-
315
- config = %{
316
- host logs.google.com
317
- port 777
318
- scheme https
319
- path /es/
320
- user john
321
- password doe
322
- template_name logstash
323
- template_file #{template_file}
324
- template_overwrite true
325
- }
326
-
327
- # connection start
328
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
329
- to_return(:status => 200, :body => "", :headers => {})
330
- # check if template exists
331
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
332
- to_return(:status => 200, :body => "", :headers => {})
333
- # creation
334
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
335
- to_return(:status => 200, :body => "", :headers => {})
336
-
337
- driver(config)
338
-
339
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
340
- end
341
-
342
-
343
- def test_template_create_invalid_filename
344
- config = %{
345
- host logs.google.com
346
- port 777
347
- scheme https
348
- path /es/
349
- user john
350
- password doe
351
- template_name logstash
352
- template_file /abc123
353
- }
354
-
355
- # connection start
356
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
357
- to_return(:status => 200, :body => "", :headers => {})
358
- # check if template exists
359
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
360
- to_return(:status => 404, :body => "", :headers => {})
361
-
362
- assert_raise(RuntimeError) {
363
- driver(config)
364
- }
365
- end
366
-
367
- def test_templates_create
368
- cwd = File.dirname(__FILE__)
369
- template_file = File.join(cwd, 'test_template.json')
370
- config = %{
371
- host logs.google.com
372
- port 777
373
- scheme https
374
- path /es/
375
- user john
376
- password doe
377
- templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
378
- }
379
-
380
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
381
- to_return(:status => 200, :body => "", :headers => {})
382
- # check if template exists
383
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
384
- to_return(:status => 404, :body => "", :headers => {})
385
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
386
- to_return(:status => 404, :body => "", :headers => {})
387
-
388
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
389
- to_return(:status => 200, :body => "", :headers => {}) #exists
390
-
391
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
392
- to_return(:status => 200, :body => "", :headers => {})
393
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
394
- to_return(:status => 200, :body => "", :headers => {})
395
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
396
- to_return(:status => 200, :body => "", :headers => {})
397
-
398
- driver(config)
399
-
400
- assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
401
- assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
402
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
403
- end
404
-
405
- def test_templates_overwrite
406
- cwd = File.dirname(__FILE__)
407
- template_file = File.join(cwd, 'test_template.json')
408
- config = %{
409
- host logs.google.com
410
- port 777
411
- scheme https
412
- path /es/
413
- user john
414
- password doe
415
- templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
416
- template_overwrite true
417
- }
418
-
419
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
420
- to_return(:status => 200, :body => "", :headers => {})
421
- # check if template exists
422
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
423
- to_return(:status => 200, :body => "", :headers => {})
424
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
425
- to_return(:status => 200, :body => "", :headers => {})
426
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
427
- to_return(:status => 200, :body => "", :headers => {}) #exists
428
-
429
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
430
- to_return(:status => 200, :body => "", :headers => {})
431
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
432
- to_return(:status => 200, :body => "", :headers => {})
433
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
434
- to_return(:status => 200, :body => "", :headers => {})
435
-
436
- driver(config)
437
-
438
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
439
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
440
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
441
- end
442
-
443
- def test_templates_not_used
444
- cwd = File.dirname(__FILE__)
445
- template_file = File.join(cwd, 'test_template.json')
446
-
447
- config = %{
448
- host logs.google.com
449
- port 777
450
- scheme https
451
- path /es/
452
- user john
453
- password doe
454
- template_name logstash
455
- template_file #{template_file}
456
- templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
457
- }
458
- # connection start
459
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
460
- to_return(:status => 200, :body => "", :headers => {})
461
- # check if template exists
462
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
463
- to_return(:status => 404, :body => "", :headers => {})
464
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
465
- to_return(:status => 404, :body => "", :headers => {})
466
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
467
- to_return(:status => 404, :body => "", :headers => {})
468
- #creation
469
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
470
- to_return(:status => 200, :body => "", :headers => {})
471
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
472
- to_return(:status => 200, :body => "", :headers => {})
473
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
474
- to_return(:status => 200, :body => "", :headers => {})
475
-
476
- driver(config)
477
-
478
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
479
-
480
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1")
481
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
482
- end
483
-
484
- def test_templates_can_be_partially_created_if_error_occurs
485
- cwd = File.dirname(__FILE__)
486
- template_file = File.join(cwd, 'test_template.json')
487
- config = %{
488
- host logs.google.com
489
- port 777
490
- scheme https
491
- path /es/
492
- user john
493
- password doe
494
- templates {"logstash1":"#{template_file}", "logstash2":"/abc" }
495
- }
496
- stub_request(:head, "https://john:doe@logs.google.com:777/es//").
497
- to_return(:status => 200, :body => "", :headers => {})
498
- # check if template exists
499
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
500
- to_return(:status => 404, :body => "", :headers => {})
501
- stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
502
- to_return(:status => 404, :body => "", :headers => {})
503
-
504
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
505
- to_return(:status => 200, :body => "", :headers => {})
506
- stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
507
- to_return(:status => 200, :body => "", :headers => {})
508
-
509
- assert_raise(RuntimeError) {
510
- driver(config)
511
- }
512
-
513
- assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
514
- assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
515
- end
516
-
517
- def test_legacy_hosts_list
518
- config = %{
519
- hosts host1:50,host2:100,host3
520
- scheme https
521
- path /es/
522
- port 123
523
- }
524
- instance = driver(config).instance
525
-
526
- assert_equal 3, instance.get_connection_options[:hosts].length
527
- host1, host2, host3 = instance.get_connection_options[:hosts]
528
-
529
- assert_equal 'host1', host1[:host]
530
- assert_equal 50, host1[:port]
531
- assert_equal 'https', host1[:scheme]
532
- assert_equal '/es/', host2[:path]
533
- assert_equal 'host3', host3[:host]
534
- assert_equal 123, host3[:port]
535
- assert_equal 'https', host3[:scheme]
536
- assert_equal '/es/', host3[:path]
537
- end
538
-
539
- def test_hosts_list
540
- config = %{
541
- hosts https://john:password@host1:443/elastic/,http://host2
542
- path /default_path
543
- user default_user
544
- password default_password
545
- }
546
- instance = driver(config).instance
547
-
548
- assert_equal 2, instance.get_connection_options[:hosts].length
549
- host1, host2 = instance.get_connection_options[:hosts]
550
-
551
- assert_equal 'host1', host1[:host]
552
- assert_equal 443, host1[:port]
553
- assert_equal 'https', host1[:scheme]
554
- assert_equal 'john', host1[:user]
555
- assert_equal 'password', host1[:password]
556
- assert_equal '/elastic/', host1[:path]
557
-
558
- assert_equal 'host2', host2[:host]
559
- assert_equal 'http', host2[:scheme]
560
- assert_equal 'default_user', host2[:user]
561
- assert_equal 'default_password', host2[:password]
562
- assert_equal '/default_path', host2[:path]
563
- end
564
-
565
- def test_hosts_list_with_escape_placeholders
566
- config = %{
567
- hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
568
- path /default_path
569
- user default_user
570
- password default_password
571
- }
572
- instance = driver(config).instance
573
-
574
- assert_equal 2, instance.get_connection_options[:hosts].length
575
- host1, host2 = instance.get_connection_options[:hosts]
576
-
577
- assert_equal 'host1', host1[:host]
578
- assert_equal 443, host1[:port]
579
- assert_equal 'https', host1[:scheme]
580
- assert_equal 'j%2Bhn', host1[:user]
581
- assert_equal 'passw%40rd', host1[:password]
582
- assert_equal '/elastic/', host1[:path]
583
-
584
- assert_equal 'host2', host2[:host]
585
- assert_equal 'http', host2[:scheme]
586
- assert_equal 'default_user', host2[:user]
587
- assert_equal 'default_password', host2[:password]
588
- assert_equal '/default_path', host2[:path]
589
- end
590
-
591
- def test_single_host_params_and_defaults
592
- config = %{
593
- host logs.google.com
594
- user john
595
- password doe
596
- }
597
- instance = driver(config).instance
598
-
599
- assert_equal 1, instance.get_connection_options[:hosts].length
600
- host1 = instance.get_connection_options[:hosts][0]
601
-
602
- assert_equal 'logs.google.com', host1[:host]
603
- assert_equal 9200, host1[:port]
604
- assert_equal 'http', host1[:scheme]
605
- assert_equal 'john', host1[:user]
606
- assert_equal 'doe', host1[:password]
607
- assert_equal nil, host1[:path]
608
- end
609
-
610
- def test_single_host_params_and_defaults_with_escape_placeholders
611
- config = %{
612
- host logs.google.com
613
- user %{j+hn}
614
- password %{d@e}
615
- }
616
- instance = driver(config).instance
617
-
618
- assert_equal 1, instance.get_connection_options[:hosts].length
619
- host1 = instance.get_connection_options[:hosts][0]
620
-
621
- assert_equal 'logs.google.com', host1[:host]
622
- assert_equal 9200, host1[:port]
623
- assert_equal 'http', host1[:scheme]
624
- assert_equal 'j%2Bhn', host1[:user]
625
- assert_equal 'd%40e', host1[:password]
626
- assert_equal nil, host1[:path]
627
- end
628
-
629
- def test_content_type_header
630
- stub_request(:head, "http://localhost:9200/").
631
- to_return(:status => 200, :body => "", :headers => {})
632
- if Elasticsearch::VERSION >= "6.0.2"
633
- elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
634
- with(headers: { "Content-Type" => "application/x-ndjson" })
635
- else
636
- elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
637
- with(headers: { "Content-Type" => "application/json" })
638
- end
639
- driver.run(default_tag: 'test') do
640
- driver.feed(sample_record)
641
- end
642
- assert_requested(elastic_request)
643
- end
644
-
645
- def test_write_message_with_bad_chunk
646
- driver.configure("target_index_key bad_value\n@log_level debug\n")
647
- stub_elastic_ping
648
- stub_elastic
649
- driver.run(default_tag: 'test') do
650
- driver.feed({'bad_value'=>"\255"})
651
- end
652
- error_log = driver.error_events.map {|e| e.last.message }
653
-
654
- assert_logs_include(error_log, /(input string invalid)|(invalid byte sequence in UTF-8)/)
655
- end
656
-
657
- def test_writes_to_default_index
658
- stub_elastic_ping
659
- stub_elastic
660
- driver.run(default_tag: 'test') do
661
- driver.feed(sample_record)
662
- end
663
- assert_equal('fluentd', index_cmds.first['index']['_index'])
664
- end
665
-
666
- def test_writes_to_default_type
667
- stub_elastic_ping
668
- stub_elastic
669
- driver.run(default_tag: 'test') do
670
- driver.feed(sample_record)
671
- end
672
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
673
- end
674
-
675
- def test_writes_to_speficied_index
676
- driver.configure("index_name myindex\n")
677
- stub_elastic_ping
678
- stub_elastic
679
- driver.run(default_tag: 'test') do
680
- driver.feed(sample_record)
681
- end
682
- assert_equal('myindex', index_cmds.first['index']['_index'])
683
- end
684
-
685
- class IndexNamePlaceholdersTest < self
686
- def test_writes_to_speficied_index_with_tag_placeholder
687
- driver.configure("index_name myindex.${tag}\n")
688
- stub_elastic_ping
689
- stub_elastic
690
- driver.run(default_tag: 'test') do
691
- driver.feed(sample_record)
692
- end
693
- assert_equal('myindex.test', index_cmds.first['index']['_index'])
694
- end
695
-
696
- def test_writes_to_speficied_index_with_time_placeholder
697
- driver.configure(Fluent::Config::Element.new(
698
- 'ROOT', '', {
699
- '@type' => 'elasticsearch',
700
- 'index_name' => 'myindex.%Y.%m.%d',
701
- }, [
702
- Fluent::Config::Element.new('buffer', 'tag,time', {
703
- 'chunk_keys' => ['tag', 'time'],
704
- 'timekey' => 3600,
705
- }, [])
706
- ]
707
- ))
708
- stub_elastic_ping
709
- stub_elastic
710
- time = Time.parse Date.today.iso8601
711
- driver.run(default_tag: 'test') do
712
- driver.feed(time.to_i, sample_record)
713
- end
714
- assert_equal("myindex.#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['index']['_index'])
715
- end
716
-
717
- def test_writes_to_speficied_index_with_custom_key_placeholder
718
- driver.configure(Fluent::Config::Element.new(
719
- 'ROOT', '', {
720
- '@type' => 'elasticsearch',
721
- 'index_name' => 'myindex.${pipeline_id}',
722
- }, [
723
- Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
724
- ]
725
- ))
726
- time = Time.parse Date.today.iso8601
727
- pipeline_id = "mypipeline"
728
- logstash_index = "myindex.#{pipeline_id}"
729
- stub_elastic_ping
730
- stub_elastic
731
- driver.run(default_tag: 'test') do
732
- driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
733
- end
734
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
735
- end
736
- end
737
-
738
- def test_writes_to_speficied_index_uppercase
739
- driver.configure("index_name MyIndex\n")
740
- stub_elastic_ping
741
- stub_elastic
742
- driver.run(default_tag: 'test') do
743
- driver.feed(sample_record)
744
- end
745
- # Allthough index_name has upper-case characters,
746
- # it should be set as lower-case when sent to elasticsearch.
747
- assert_equal('myindex', index_cmds.first['index']['_index'])
748
- end
749
-
750
- def test_writes_to_target_index_key
751
- driver.configure("target_index_key @target_index\n")
752
- stub_elastic_ping
753
- stub_elastic
754
- record = sample_record.clone
755
- driver.run(default_tag: 'test') do
756
- driver.feed(sample_record.merge('@target_index' => 'local-override'))
757
- end
758
- assert_equal('local-override', index_cmds.first['index']['_index'])
759
- assert_nil(index_cmds[1]['@target_index'])
760
- end
761
-
762
- def test_writes_to_target_index_key_logstash
763
- driver.configure("target_index_key @target_index
764
- logstash_format true")
765
- time = Time.parse Date.today.iso8601
766
- stub_elastic_ping
767
- stub_elastic
768
- driver.run(default_tag: 'test') do
769
- driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
770
- end
771
- assert_equal('local-override', index_cmds.first['index']['_index'])
772
- end
773
-
774
- def test_writes_to_target_index_key_logstash_uppercase
775
- driver.configure("target_index_key @target_index
776
- logstash_format true")
777
- time = Time.parse Date.today.iso8601
778
- stub_elastic_ping
779
- stub_elastic
780
- driver.run(default_tag: 'test') do
781
- driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
782
- end
783
- # Allthough @target_index has upper-case characters,
784
- # it should be set as lower-case when sent to elasticsearch.
785
- assert_equal('local-override', index_cmds.first['index']['_index'])
786
- end
787
-
788
- def test_writes_to_default_index_with_pipeline
789
- pipeline = "fluentd"
790
- driver.configure("pipeline #{pipeline}")
791
- stub_elastic_ping
792
- stub_elastic
793
- driver.run(default_tag: 'test') do
794
- driver.feed(sample_record)
795
- end
796
- assert_equal(pipeline, index_cmds.first['index']['pipeline'])
797
- end
798
-
799
- def test_writes_to_target_index_key_fallack
800
- driver.configure("target_index_key @target_index\n")
801
- stub_elastic_ping
802
- stub_elastic
803
- driver.run(default_tag: 'test') do
804
- driver.feed(sample_record)
805
- end
806
- assert_equal('fluentd', index_cmds.first['index']['_index'])
807
- end
808
-
809
- def test_writes_to_target_index_key_fallack_logstash
810
- driver.configure("target_index_key @target_index\n
811
- logstash_format true")
812
- time = Time.parse Date.today.iso8601
813
- logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
814
- stub_elastic_ping
815
- stub_elastic
816
- driver.run(default_tag: 'test') do
817
- driver.feed(time.to_i, sample_record)
818
- end
819
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
820
- end
821
-
822
- data("border" => {"es_version" => 6, "_type" => "mytype"},
823
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
824
- )
825
- def test_writes_to_speficied_type(data)
826
- driver('', data["es_version"]).configure("type_name mytype\n")
827
- stub_elastic_ping
828
- stub_elastic
829
- driver.run(default_tag: 'test') do
830
- driver.feed(sample_record)
831
- end
832
- assert_equal(data['_type'], index_cmds.first['index']['_type'])
833
- end
834
-
835
- data("border" => {"es_version" => 6, "_type" => "mytype.test"},
836
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
837
- )
838
- def test_writes_to_speficied_type_with_placeholders(data)
839
- driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
840
- stub_elastic_ping
841
- stub_elastic
842
- driver.run(default_tag: 'test') do
843
- driver.feed(sample_record)
844
- end
845
- assert_equal(data['_type'], index_cmds.first['index']['_type'])
846
- end
847
-
848
- data("old" => {"es_version" => 2, "_type" => "local-override"},
849
- "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
850
- "border" => {"es_version" => 6, "_type" => "fluentd"},
851
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
852
- )
853
- def test_writes_to_target_type_key(data)
854
- driver('', data["es_version"]).configure("target_type_key @target_type\n")
855
- stub_elastic_ping
856
- stub_elastic
857
- record = sample_record.clone
858
- driver.run(default_tag: 'test') do
859
- driver.feed(sample_record.merge('@target_type' => 'local-override'))
860
- end
861
- assert_equal(data["_type"], index_cmds.first['index']['_type'])
862
- assert_nil(index_cmds[1]['@target_type'])
863
- end
864
-
865
- def test_writes_to_target_type_key_fallack_to_default
866
- driver.configure("target_type_key @target_type\n")
867
- stub_elastic_ping
868
- stub_elastic
869
- driver.run(default_tag: 'test') do
870
- driver.feed(sample_record)
871
- end
872
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
873
- end
874
-
875
- def test_writes_to_target_type_key_fallack_to_type_name
876
- driver.configure("target_type_key @target_type
877
- type_name mytype")
878
- stub_elastic_ping
879
- stub_elastic
880
- driver.run(default_tag: 'test') do
881
- driver.feed(sample_record)
882
- end
883
- assert_equal('mytype', index_cmds.first['index']['_type'])
884
- end
885
-
886
- data("old" => {"es_version" => 2, "_type" => "local-override"},
887
- "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
888
- "border" => {"es_version" => 6, "_type" => "fluentd"},
889
- "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
890
- )
891
- def test_writes_to_target_type_key_nested(data)
892
- driver('', data["es_version"]).configure("target_type_key kubernetes.labels.log_type\n")
893
- stub_elastic_ping
894
- stub_elastic
895
- driver.run(default_tag: 'test') do
896
- driver.feed(sample_record.merge('kubernetes' => {
897
- 'labels' => {
898
- 'log_type' => 'local-override'
899
- }
900
- }))
901
- end
902
- assert_equal(data["_type"], index_cmds.first['index']['_type'])
903
- assert_nil(index_cmds[1]['kubernetes']['labels']['log_type'])
904
- end
905
-
906
- def test_writes_to_target_type_key_fallack_to_default_nested
907
- driver.configure("target_type_key kubernetes.labels.log_type\n")
908
- stub_elastic_ping
909
- stub_elastic
910
- driver.run(default_tag: 'test') do
911
- driver.feed(sample_record.merge('kubernetes' => {
912
- 'labels' => {
913
- 'other_labels' => 'test'
914
- }
915
- }))
916
- end
917
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
918
- end
919
-
920
- def test_writes_to_speficied_host
921
- driver.configure("host 192.168.33.50\n")
922
- stub_elastic_ping("http://192.168.33.50:9200")
923
- elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
924
- driver.run(default_tag: 'test') do
925
- driver.feed(sample_record)
926
- end
927
- assert_requested(elastic_request)
928
- end
929
-
930
- def test_writes_to_speficied_port
931
- driver.configure("port 9201\n")
932
- stub_elastic_ping("http://localhost:9201")
933
- elastic_request = stub_elastic("http://localhost:9201/_bulk")
934
- driver.run(default_tag: 'test') do
935
- driver.feed(sample_record)
936
- end
937
- assert_requested(elastic_request)
938
- end
939
-
940
- def test_writes_to_multi_hosts
941
- hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
942
- hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
943
-
944
- driver.configure("hosts #{hosts_string}")
945
-
946
- hosts.each do |host_info|
947
- host, port = host_info
948
- stub_elastic_ping("http://#{host}:#{port}")
949
- stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
950
- end
951
-
952
- driver.run(default_tag: 'test') do
953
- 1000.times do
954
- driver.feed(sample_record.merge('age'=>rand(100)))
955
- end
956
- end
957
-
958
- # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
959
- # it's Fluentd test driver's constraint
960
- # so @index_command_counts.size is always 1
961
-
962
- assert(@index_command_counts.size > 0, "not working with hosts options")
963
-
964
- total = 0
965
- @index_command_counts.each do |url, count|
966
- total += count
967
- end
968
- assert_equal(2000, total)
969
- end
970
-
971
- def test_nested_record_with_flattening_on
972
- driver.configure("flatten_hashes true
973
- flatten_hashes_separator |")
974
-
975
- original_hash = {"foo" => {"bar" => "baz"}, "people" => [
976
- {"age" => "25", "height" => "1ft"},
977
- {"age" => "30", "height" => "2ft"}
978
- ]}
979
-
980
- expected_output = {"foo|bar"=>"baz", "people" => [
981
- {"age" => "25", "height" => "1ft"},
982
- {"age" => "30", "height" => "2ft"}
983
- ]}
984
-
985
- stub_elastic_ping
986
- stub_elastic
987
- driver.run(default_tag: 'test') do
988
- driver.feed(original_hash)
989
- end
990
- assert_equal expected_output, index_cmds[1]
991
- end
992
-
993
- def test_nested_record_with_flattening_off
994
- # flattening off by default
995
-
996
- original_hash = {"foo" => {"bar" => "baz"}}
997
- expected_output = {"foo" => {"bar" => "baz"}}
998
-
999
- stub_elastic_ping
1000
- stub_elastic
1001
- driver.run(default_tag: 'test') do
1002
- driver.feed(original_hash)
1003
- end
1004
- assert_equal expected_output, index_cmds[1]
1005
- end
1006
-
1007
- def test_makes_bulk_request
1008
- stub_elastic_ping
1009
- stub_elastic
1010
- driver.run(default_tag: 'test') do
1011
- driver.feed(sample_record)
1012
- driver.feed(sample_record.merge('age' => 27))
1013
- end
1014
- assert_equal(4, index_cmds.count)
1015
- end
1016
-
1017
- def test_all_records_are_preserved_in_bulk
1018
- stub_elastic_ping
1019
- stub_elastic
1020
- driver.run(default_tag: 'test') do
1021
- driver.feed(sample_record)
1022
- driver.feed(sample_record.merge('age' => 27))
1023
- end
1024
- assert_equal(26, index_cmds[1]['age'])
1025
- assert_equal(27, index_cmds[3]['age'])
1026
- end
1027
-
1028
- def test_writes_to_logstash_index
1029
- driver.configure("logstash_format true\n")
1030
- #
1031
- # This is 1 second past midnight in BST, so the UTC index should be the day before
1032
- dt = DateTime.new(2015, 6, 1, 0, 0, 1, "+01:00")
1033
- logstash_index = "logstash-2015.05.31"
1034
- stub_elastic_ping
1035
- stub_elastic
1036
- driver.run(default_tag: 'test') do
1037
- driver.feed(dt.to_time.to_i, sample_record)
1038
- end
1039
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1040
- end
1041
-
1042
- def test_writes_to_logstash_non_utc_index
1043
- driver.configure("logstash_format true
1044
- utc_index false")
1045
- # When using `utc_index false` the index time will be the local day of
1046
- # ingestion time
1047
- time = Date.today.to_time
1048
- index = "logstash-#{time.strftime("%Y.%m.%d")}"
1049
- stub_elastic_ping
1050
- stub_elastic
1051
- driver.run(default_tag: 'test') do
1052
- driver.feed(time.to_i, sample_record)
1053
- end
1054
- assert_equal(index, index_cmds.first['index']['_index'])
1055
- end
1056
-
1057
- def test_writes_to_logstash_index_with_specified_prefix
1058
- driver.configure("logstash_format true
1059
- logstash_prefix myprefix")
1060
- time = Time.parse Date.today.iso8601
1061
- logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1062
- stub_elastic_ping
1063
- stub_elastic
1064
- driver.run(default_tag: 'test') do
1065
- driver.feed(time.to_i, sample_record)
1066
- end
1067
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1068
- end
1069
-
1070
- def test_writes_to_logstash_index_with_specified_prefix_and_separator
1071
- separator = '_'
1072
- driver.configure("logstash_format true
1073
- logstash_prefix_separator #{separator}
1074
- logstash_prefix myprefix")
1075
- time = Time.parse Date.today.iso8601
1076
- logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
1077
- stub_elastic_ping
1078
- stub_elastic
1079
- driver.run(default_tag: 'test') do
1080
- driver.feed(time.to_i, sample_record)
1081
- end
1082
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1083
- end
1084
-
1085
- class LogStashPrefixPlaceholdersTest < self
1086
- def test_writes_to_logstash_index_with_specified_prefix_and_tag_placeholder
1087
- driver.configure("logstash_format true
1088
- logstash_prefix myprefix-${tag}")
1089
- time = Time.parse Date.today.iso8601
1090
- logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
1091
- stub_elastic_ping
1092
- stub_elastic
1093
- driver.run(default_tag: 'test') do
1094
- driver.feed(time.to_i, sample_record)
1095
- end
1096
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1097
- end
1098
-
1099
- def test_writes_to_logstash_index_with_specified_prefix_and_time_placeholder
1100
- driver.configure(Fluent::Config::Element.new(
1101
- 'ROOT', '', {
1102
- '@type' => 'elasticsearch',
1103
- 'logstash_format' => true,
1104
- 'logstash_prefix' => 'myprefix-%H',
1105
- }, [
1106
- Fluent::Config::Element.new('buffer', 'tag,time', {
1107
- 'chunk_keys' => ['tag', 'time'],
1108
- 'timekey' => 3600,
1109
- }, [])
1110
- ]
1111
- ))
1112
- time = Time.parse Date.today.iso8601
1113
- logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
1114
- stub_elastic_ping
1115
- stub_elastic
1116
- driver.run(default_tag: 'test') do
1117
- driver.feed(time.to_i, sample_record)
1118
- end
1119
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1120
- end
1121
-
1122
- def test_writes_to_logstash_index_with_specified_prefix_and_custom_key_placeholder
1123
- driver.configure(Fluent::Config::Element.new(
1124
- 'ROOT', '', {
1125
- '@type' => 'elasticsearch',
1126
- 'logstash_format' => true,
1127
- 'logstash_prefix' => 'myprefix-${pipeline_id}',
1128
- }, [
1129
- Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
1130
- ]
1131
- ))
1132
- time = Time.parse Date.today.iso8601
1133
- pipeline_id = "mypipeline"
1134
- logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
1135
- stub_elastic_ping
1136
- stub_elastic
1137
- driver.run(default_tag: 'test') do
1138
- driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
1139
- end
1140
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1141
- end
1142
- end
1143
-
1144
- def test_writes_to_logstash_index_with_specified_prefix_uppercase
1145
- driver.configure("logstash_format true
1146
- logstash_prefix MyPrefix")
1147
- time = Time.parse Date.today.iso8601
1148
- logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1149
- stub_elastic_ping
1150
- stub_elastic
1151
- driver.run(default_tag: 'test') do
1152
- driver.feed(time.to_i, sample_record)
1153
- end
1154
- # Allthough logstash_prefix has upper-case characters,
1155
- # it should be set as lower-case when sent to elasticsearch.
1156
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1157
- end
1158
-
1159
- def test_writes_to_logstash_index_with_specified_dateformat
1160
- driver.configure("logstash_format true
1161
- logstash_dateformat %Y.%m")
1162
- time = Time.parse Date.today.iso8601
1163
- logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
1164
- stub_elastic_ping
1165
- stub_elastic
1166
- driver.run(default_tag: 'test') do
1167
- driver.feed(time.to_i, sample_record)
1168
- end
1169
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1170
- end
1171
-
1172
- def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
1173
- driver.configure("logstash_format true
1174
- logstash_prefix myprefix
1175
- logstash_dateformat %Y.%m")
1176
- time = Time.parse Date.today.iso8601
1177
- logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
1178
- stub_elastic_ping
1179
- stub_elastic
1180
- driver.run(default_tag: 'test') do
1181
- driver.feed(time.to_i, sample_record)
1182
- end
1183
- assert_equal(logstash_index, index_cmds.first['index']['_index'])
1184
- end
1185
-
1186
- def test_error_if_tag_not_in_chunk_keys
1187
- assert_raise(Fluent::ConfigError) {
1188
- config = %{
1189
- <buffer foo>
1190
- </buffer>
1191
- }
1192
- driver.configure(config)
1193
- }
1194
- end
1195
-
1196
- def test_can_use_custom_chunk_along_with_tag
1197
- config = %{
1198
- <buffer tag, foo>
1199
- </buffer>
1200
- }
1201
- driver.configure(config)
1202
- end
1203
-
1204
- def test_doesnt_add_logstash_timestamp_by_default
1205
- stub_elastic_ping
1206
- stub_elastic
1207
- driver.run(default_tag: 'test') do
1208
- driver.feed(sample_record)
1209
- end
1210
- assert_nil(index_cmds[1]['@timestamp'])
1211
- end
1212
-
1213
- def test_adds_timestamp_when_logstash
1214
- driver.configure("logstash_format true\n")
1215
- stub_elastic_ping
1216
- stub_elastic
1217
- ts = DateTime.now
1218
- time = Fluent::EventTime.from_time(ts.to_time)
1219
- driver.run(default_tag: 'test') do
1220
- driver.feed(time, sample_record)
1221
- end
1222
- assert(index_cmds[1].has_key? '@timestamp')
1223
- assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1224
- end
1225
-
1226
- def test_adds_timestamp_when_include_timestamp
1227
- driver.configure("include_timestamp true\n")
1228
- stub_elastic_ping
1229
- stub_elastic
1230
- ts = DateTime.now
1231
- time = Fluent::EventTime.from_time(ts.to_time)
1232
- driver.run(default_tag: 'test') do
1233
- driver.feed(time, sample_record)
1234
- end
1235
- assert(index_cmds[1].has_key? '@timestamp')
1236
- assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1237
- end
1238
-
1239
- def test_uses_custom_timestamp_when_included_in_record
1240
- driver.configure("logstash_format true\n")
1241
- stub_elastic_ping
1242
- stub_elastic
1243
- ts = DateTime.new(2001,2,3).iso8601
1244
- driver.run(default_tag: 'test') do
1245
- driver.feed(sample_record.merge!('@timestamp' => ts))
1246
- end
1247
- assert(index_cmds[1].has_key? '@timestamp')
1248
- assert_equal(index_cmds[1]['@timestamp'], ts)
1249
- end
1250
-
1251
- def test_uses_custom_timestamp_when_included_in_record_without_logstash
1252
- driver.configure("include_timestamp true\n")
1253
- stub_elastic_ping
1254
- stub_elastic
1255
- ts = DateTime.new(2001,2,3).iso8601
1256
- driver.run(default_tag: 'test') do
1257
- driver.feed(sample_record.merge!('@timestamp' => ts))
1258
- end
1259
- assert(index_cmds[1].has_key? '@timestamp')
1260
- assert_equal(index_cmds[1]['@timestamp'], ts)
1261
- end
1262
-
1263
- def test_uses_custom_time_key
1264
- driver.configure("logstash_format true
1265
- time_key vtm\n")
1266
- stub_elastic_ping
1267
- stub_elastic
1268
- ts = DateTime.new(2001,2,3).iso8601(9)
1269
- driver.run(default_tag: 'test') do
1270
- driver.feed(sample_record.merge!('vtm' => ts))
1271
- end
1272
- assert(index_cmds[1].has_key? '@timestamp')
1273
- assert_equal(index_cmds[1]['@timestamp'], ts)
1274
- end
1275
-
1276
- def test_uses_custom_time_key_with_format
1277
- driver.configure("logstash_format true
1278
- time_key_format %Y-%m-%d %H:%M:%S.%N%z
1279
- time_key vtm\n")
1280
- stub_elastic_ping
1281
- stub_elastic
1282
- ts = "2001-02-03 13:14:01.673+02:00"
1283
- driver.run(default_tag: 'test') do
1284
- driver.feed(sample_record.merge!('vtm' => ts))
1285
- end
1286
- assert(index_cmds[1].has_key? '@timestamp')
1287
- assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1288
- assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1289
- end
1290
-
1291
- def test_uses_custom_time_key_with_format_without_logstash
1292
- driver.configure("include_timestamp true
1293
- index_name test
1294
- time_key_format %Y-%m-%d %H:%M:%S.%N%z
1295
- time_key vtm\n")
1296
- stub_elastic_ping
1297
- stub_elastic
1298
- ts = "2001-02-03 13:14:01.673+02:00"
1299
- driver.run(default_tag: 'test') do
1300
- driver.feed(sample_record.merge!('vtm' => ts))
1301
- end
1302
- assert(index_cmds[1].has_key? '@timestamp')
1303
- assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1304
- assert_equal("test", index_cmds[0]['index']['_index'])
1305
- end
1306
-
1307
- def test_uses_custom_time_key_exclude_timekey
1308
- driver.configure("logstash_format true
1309
- time_key vtm
1310
- time_key_exclude_timestamp true\n")
1311
- stub_elastic_ping
1312
- stub_elastic
1313
- ts = DateTime.new(2001,2,3).iso8601
1314
- driver.run(default_tag: 'test') do
1315
- driver.feed(sample_record.merge!('vtm' => ts))
1316
- end
1317
- assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be messing')
1318
- end
1319
-
1320
- def test_uses_custom_time_key_format
1321
- driver.configure("logstash_format true
1322
- time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1323
- stub_elastic_ping
1324
- stub_elastic
1325
- ts = "2001-02-03T13:14:01.673+02:00"
1326
- driver.run(default_tag: 'test') do
1327
- driver.feed(sample_record.merge!('@timestamp' => ts))
1328
- end
1329
- assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1330
- assert(index_cmds[1].has_key? '@timestamp')
1331
- assert_equal(index_cmds[1]['@timestamp'], ts)
1332
- end
1333
-
1334
- def test_uses_custom_time_key_format_without_logstash
1335
- driver.configure("include_timestamp true
1336
- index_name test
1337
- time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1338
- stub_elastic_ping
1339
- stub_elastic
1340
- ts = "2001-02-03T13:14:01.673+02:00"
1341
- driver.run(default_tag: 'test') do
1342
- driver.feed(sample_record.merge!('@timestamp' => ts))
1343
- end
1344
- assert_equal("test", index_cmds[0]['index']['_index'])
1345
- assert(index_cmds[1].has_key? '@timestamp')
1346
- assert_equal(index_cmds[1]['@timestamp'], ts)
1347
- end
1348
-
1349
- data(:default => nil,
1350
- :custom_tag => 'es_plugin.output.time.error')
1351
- def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
1352
- tag_config = tag_for_error ? "time_parse_error_tag #{tag_for_error}" : ''
1353
- tag_for_error = 'Fluent::ElasticsearchOutput::TimeParser.error' if tag_for_error.nil?
1354
- driver.configure("logstash_format true
1355
- time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n#{tag_config}\n")
1356
- stub_elastic_ping
1357
- stub_elastic
1358
-
1359
- ts = "2001/02/03 13:14:01,673+02:00"
1360
- index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
1361
-
1362
- flexmock(driver.instance.router).should_receive(:emit_error_event)
1363
- .with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once
1364
- driver.run(default_tag: 'test') do
1365
- driver.feed(sample_record.merge!('@timestamp' => ts))
1366
- end
1367
-
1368
- assert_equal(index, index_cmds[0]['index']['_index'])
1369
- assert(index_cmds[1].has_key? '@timestamp')
1370
- assert_equal(index_cmds[1]['@timestamp'], ts)
1371
- end
1372
-
1373
-
1374
- def test_uses_custom_time_key_format_obscure_format
1375
- driver.configure("logstash_format true
1376
- time_key_format %a %b %d %H:%M:%S %Z %Y\n")
1377
- stub_elastic_ping
1378
- stub_elastic
1379
- ts = "Thu Nov 29 14:33:20 GMT 2001"
1380
- driver.run(default_tag: 'test') do
1381
- driver.feed(sample_record.merge!('@timestamp' => ts))
1382
- end
1383
- assert_equal("logstash-2001.11.29", index_cmds[0]['index']['_index'])
1384
- assert(index_cmds[1].has_key? '@timestamp')
1385
- assert_equal(index_cmds[1]['@timestamp'], ts)
1386
- end
1387
-
1388
- def test_uses_nanosecond_precision_by_default
1389
- driver.configure("logstash_format true\n")
1390
- stub_elastic_ping
1391
- stub_elastic
1392
- time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1393
- driver.run(default_tag: 'test') do
1394
- driver.feed(time, sample_record)
1395
- end
1396
- assert(index_cmds[1].has_key? '@timestamp')
1397
- assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(9))
1398
- end
1399
-
1400
- def test_uses_subsecond_precision_when_configured
1401
- driver.configure("logstash_format true
1402
- time_precision 3\n")
1403
- stub_elastic_ping
1404
- stub_elastic
1405
- time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1406
- driver.run(default_tag: 'test') do
1407
- driver.feed(time, sample_record)
1408
- end
1409
- assert(index_cmds[1].has_key? '@timestamp')
1410
- assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(3))
1411
- end
1412
-
1413
- def test_doesnt_add_tag_key_by_default
1414
- stub_elastic_ping
1415
- stub_elastic
1416
- driver.run(default_tag: 'test') do
1417
- driver.feed(sample_record)
1418
- end
1419
- assert_nil(index_cmds[1]['tag'])
1420
- end
1421
-
1422
- def test_adds_tag_key_when_configured
1423
- driver.configure("include_tag_key true\n")
1424
- stub_elastic_ping
1425
- stub_elastic
1426
- driver.run(default_tag: 'mytag') do
1427
- driver.feed(sample_record)
1428
- end
1429
- assert(index_cmds[1].has_key?('tag'))
1430
- assert_equal(index_cmds[1]['tag'], 'mytag')
1431
- end
1432
-
1433
- def test_adds_id_key_when_configured
1434
- driver.configure("id_key request_id\n")
1435
- stub_elastic_ping
1436
- stub_elastic
1437
- driver.run(default_tag: 'test') do
1438
- driver.feed(sample_record)
1439
- end
1440
- assert_equal(index_cmds[0]['index']['_id'], '42')
1441
- end
1442
-
1443
- class NestedIdKeyTest < self
1444
- def test_adds_nested_id_key_with_dot
1445
- driver.configure("id_key nested.request_id\n")
1446
- stub_elastic_ping
1447
- stub_elastic
1448
- driver.run(default_tag: 'test') do
1449
- driver.feed(nested_sample_record)
1450
- end
1451
- assert_equal(index_cmds[0]['index']['_id'], '42')
1452
- end
1453
-
1454
- def test_adds_nested_id_key_with_dollar_dot
1455
- driver.configure("id_key $.nested.request_id\n")
1456
- stub_elastic_ping
1457
- stub_elastic
1458
- driver.run(default_tag: 'test') do
1459
- driver.feed(nested_sample_record)
1460
- end
1461
- assert_equal(index_cmds[0]['index']['_id'], '42')
1462
- end
1463
-
1464
- def test_adds_nested_id_key_with_bracket
1465
- driver.configure("id_key $['nested']['request_id']\n")
1466
- stub_elastic_ping
1467
- stub_elastic
1468
- driver.run(default_tag: 'test') do
1469
- driver.feed(nested_sample_record)
1470
- end
1471
- assert_equal(index_cmds[0]['index']['_id'], '42')
1472
- end
1473
- end
1474
-
1475
- def test_doesnt_add_id_key_if_missing_when_configured
1476
- driver.configure("id_key another_request_id\n")
1477
- stub_elastic_ping
1478
- stub_elastic
1479
- driver.run(default_tag: 'test') do
1480
- driver.feed(sample_record)
1481
- end
1482
- assert(!index_cmds[0]['index'].has_key?('_id'))
1483
- end
1484
-
1485
- def test_adds_id_key_when_not_configured
1486
- stub_elastic_ping
1487
- stub_elastic
1488
- driver.run(default_tag: 'test') do
1489
- driver.feed(sample_record)
1490
- end
1491
- assert(!index_cmds[0]['index'].has_key?('_id'))
1492
- end
1493
-
1494
- def test_adds_parent_key_when_configured
1495
- driver.configure("parent_key parent_id\n")
1496
- stub_elastic_ping
1497
- stub_elastic
1498
- driver.run(default_tag: 'test') do
1499
- driver.feed(sample_record)
1500
- end
1501
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1502
- end
1503
-
1504
- class NestedParentKeyTest < self
1505
- def test_adds_nested_parent_key_with_dot
1506
- driver.configure("parent_key nested.parent_id\n")
1507
- stub_elastic_ping
1508
- stub_elastic
1509
- driver.run(default_tag: 'test') do
1510
- driver.feed(nested_sample_record)
1511
- end
1512
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1513
- end
1514
-
1515
- def test_adds_nested_parent_key_with_dollar_dot
1516
- driver.configure("parent_key $.nested.parent_id\n")
1517
- stub_elastic_ping
1518
- stub_elastic
1519
- driver.run(default_tag: 'test') do
1520
- driver.feed(nested_sample_record)
1521
- end
1522
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1523
- end
1524
-
1525
- def test_adds_nested_parent_key_with_bracket
1526
- driver.configure("parent_key $['nested']['parent_id']\n")
1527
- stub_elastic_ping
1528
- stub_elastic
1529
- driver.run(default_tag: 'test') do
1530
- driver.feed(nested_sample_record)
1531
- end
1532
- assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1533
- end
1534
- end
1535
-
1536
- def test_doesnt_add_parent_key_if_missing_when_configured
1537
- driver.configure("parent_key another_parent_id\n")
1538
- stub_elastic_ping
1539
- stub_elastic
1540
- driver.run(default_tag: 'test') do
1541
- driver.feed(sample_record)
1542
- end
1543
- assert(!index_cmds[0]['index'].has_key?('_parent'))
1544
- end
1545
-
1546
- def test_adds_parent_key_when_not_configured
1547
- stub_elastic_ping
1548
- stub_elastic
1549
- driver.run(default_tag: 'test') do
1550
- driver.feed(sample_record)
1551
- end
1552
- assert(!index_cmds[0]['index'].has_key?('_parent'))
1553
- end
1554
-
1555
- def test_adds_routing_key_when_configured
1556
- driver.configure("routing_key routing_id\n")
1557
- stub_elastic_ping
1558
- stub_elastic
1559
- driver.run(default_tag: 'test') do
1560
- driver.feed(sample_record)
1561
- end
1562
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1563
- end
1564
-
1565
- class NestedRoutingKeyTest < self
1566
- def test_adds_nested_routing_key_with_dot
1567
- driver.configure("routing_key nested.routing_id\n")
1568
- stub_elastic_ping
1569
- stub_elastic
1570
- driver.run(default_tag: 'test') do
1571
- driver.feed(nested_sample_record)
1572
- end
1573
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1574
- end
1575
-
1576
- def test_adds_nested_routing_key_with_dollar_dot
1577
- driver.configure("routing_key $.nested.routing_id\n")
1578
- stub_elastic_ping
1579
- stub_elastic
1580
- driver.run(default_tag: 'test') do
1581
- driver.feed(nested_sample_record)
1582
- end
1583
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1584
- end
1585
-
1586
- def test_adds_nested_routing_key_with_bracket
1587
- driver.configure("routing_key $['nested']['routing_id']\n")
1588
- stub_elastic_ping
1589
- stub_elastic
1590
- driver.run(default_tag: 'test') do
1591
- driver.feed(nested_sample_record)
1592
- end
1593
- assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1594
- end
1595
- end
1596
-
1597
- def test_doesnt_add_routing_key_if_missing_when_configured
1598
- driver.configure("routing_key another_routing_id\n")
1599
- stub_elastic_ping
1600
- stub_elastic
1601
- driver.run(default_tag: 'test') do
1602
- driver.feed(sample_record)
1603
- end
1604
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1605
- end
1606
-
1607
- def test_adds_routing_key_when_not_configured
1608
- stub_elastic_ping
1609
- stub_elastic
1610
- driver.run(default_tag: 'test') do
1611
- driver.feed(sample_record)
1612
- end
1613
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1614
- end
1615
-
1616
- def test_remove_one_key
1617
- driver.configure("remove_keys key1\n")
1618
- stub_elastic_ping
1619
- stub_elastic
1620
- driver.run(default_tag: 'test') do
1621
- driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1622
- end
1623
- assert(!index_cmds[1].has_key?('key1'))
1624
- assert(index_cmds[1].has_key?('key2'))
1625
- end
1626
-
1627
- def test_remove_multi_keys
1628
- driver.configure("remove_keys key1, key2\n")
1629
- stub_elastic_ping
1630
- stub_elastic
1631
- driver.run(default_tag: 'test') do
1632
- driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1633
- end
1634
- assert(!index_cmds[1].has_key?('key1'))
1635
- assert(!index_cmds[1].has_key?('key2'))
1636
- end
1637
-
1638
- def test_request_error
1639
- stub_elastic_ping
1640
- stub_elastic_unavailable
1641
- assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
1642
- driver.run(default_tag: 'test') do
1643
- driver.feed(sample_record)
1644
- end
1645
- }
1646
- end
1647
-
1648
- def test_connection_failed_retry
1649
- connection_resets = 0
1650
-
1651
- stub_elastic_ping(url="http://localhost:9200").with do |req|
1652
- connection_resets += 1
1653
- end
1654
-
1655
- stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1656
- raise Faraday::ConnectionFailed, "Test message"
1657
- end
1658
-
1659
- driver.run(default_tag: 'test') do
1660
- driver.feed(sample_record)
1661
- end
1662
- assert_equal(connection_resets, 3)
1663
- end
1664
-
1665
- def test_reconnect_on_error_enabled
1666
- connection_resets = 0
1667
-
1668
- stub_elastic_ping(url="http://localhost:9200").with do |req|
1669
- connection_resets += 1
1670
- end
1671
-
1672
- stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1673
- raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1674
- end
1675
-
1676
- driver.configure("reconnect_on_error true\n")
1677
-
1678
- assert_raise(ZeroDivisionError) {
1679
- driver.run(default_tag: 'test', shutdown: false) do
1680
- driver.feed(sample_record)
1681
- end
1682
- }
1683
-
1684
- assert_raise(Timeout::Error) {
1685
- driver.run(default_tag: 'test', shutdown: false) do
1686
- driver.feed(sample_record)
1687
- end
1688
- }
1689
- # FIXME: Consider keywords arguments in #run and how to test this later.
1690
- # Because v0.14 test driver does not have 1 to 1 correspondence between #run and #flush in tests.
1691
- assert_equal(connection_resets, 1)
1692
- end
1693
-
1694
- def test_reconnect_on_error_disabled
1695
- connection_resets = 0
1696
-
1697
- stub_elastic_ping(url="http://localhost:9200").with do |req|
1698
- connection_resets += 1
1699
- end
1700
-
1701
- stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1702
- raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1703
- end
1704
-
1705
- driver.configure("reconnect_on_error false\n")
1706
-
1707
- assert_raise(ZeroDivisionError) {
1708
- driver.run(default_tag: 'test', shutdown: false) do
1709
- driver.feed(sample_record)
1710
- end
1711
- }
1712
-
1713
- assert_raise(Timeout::Error) {
1714
- driver.run(default_tag: 'test', shutdown: false) do
1715
- driver.feed(sample_record)
1716
- end
1717
- }
1718
- assert_equal(connection_resets, 1)
1719
- end
1720
-
1721
- def test_bulk_error_retags_when_configured
1722
- driver.configure("retry_tag retry\n")
1723
- stub_elastic_ping
1724
- stub_request(:post, 'http://localhost:9200/_bulk')
1725
- .to_return(lambda do |req|
1726
- { :status => 200,
1727
- :headers => { 'Content-Type' => 'json' },
1728
- :body => %({
1729
- "took" : 1,
1730
- "errors" : true,
1731
- "items" : [
1732
- {
1733
- "create" : {
1734
- "_index" : "foo",
1735
- "_type" : "bar",
1736
- "_id" : "abc",
1737
- "status" : 500,
1738
- "error" : {
1739
- "type" : "some unrecognized type",
1740
- "reason":"some error to cause version mismatch"
1741
- }
1742
- }
1743
- }
1744
- ]
1745
- })
1746
- }
1747
- end)
1748
-
1749
- driver.run(default_tag: 'test') do
1750
- driver.feed(1, sample_record)
1751
- end
1752
-
1753
- assert_equal [['retry', 1, sample_record]], driver.events
1754
- end
1755
-
1756
- def test_bulk_error
1757
- stub_elastic_ping
1758
- stub_request(:post, 'http://localhost:9200/_bulk')
1759
- .to_return(lambda do |req|
1760
- { :status => 200,
1761
- :headers => { 'Content-Type' => 'json' },
1762
- :body => %({
1763
- "took" : 1,
1764
- "errors" : true,
1765
- "items" : [
1766
- {
1767
- "create" : {
1768
- "_index" : "foo",
1769
- "_type" : "bar",
1770
- "_id" : "abc",
1771
- "status" : 500,
1772
- "error" : {
1773
- "type" : "some unrecognized type",
1774
- "reason":"some error to cause version mismatch"
1775
- }
1776
- }
1777
- },
1778
- {
1779
- "create" : {
1780
- "_index" : "foo",
1781
- "_type" : "bar",
1782
- "_id" : "abc",
1783
- "status" : 201
1784
- }
1785
- },
1786
- {
1787
- "create" : {
1788
- "_index" : "foo",
1789
- "_type" : "bar",
1790
- "_id" : "abc",
1791
- "status" : 500,
1792
- "error" : {
1793
- "type" : "some unrecognized type",
1794
- "reason":"some error to cause version mismatch"
1795
- }
1796
- }
1797
- },
1798
- {
1799
- "create" : {
1800
- "_index" : "foo",
1801
- "_type" : "bar",
1802
- "_id" : "abc",
1803
- "_id" : "abc",
1804
- "status" : 409
1805
- }
1806
- }
1807
- ]
1808
- })
1809
- }
1810
- end)
1811
-
1812
- driver.run(default_tag: 'test') do
1813
- driver.feed(1, sample_record)
1814
- driver.feed(2, sample_record)
1815
- driver.feed(3, sample_record)
1816
- driver.feed(4, sample_record)
1817
- end
1818
-
1819
- expect = [['test', 1, sample_record],
1820
- ['test', 3, sample_record]]
1821
- assert_equal expect, driver.events
1822
- end
1823
-
1824
- def test_update_should_not_write_if_theres_no_id
1825
- driver.configure("write_operation update\n")
1826
- stub_elastic_ping
1827
- stub_elastic
1828
- driver.run(default_tag: 'test') do
1829
- driver.feed(sample_record)
1830
- end
1831
- assert_nil(index_cmds)
1832
- end
1833
-
1834
- def test_upsert_should_not_write_if_theres_no_id
1835
- driver.configure("write_operation upsert\n")
1836
- stub_elastic_ping
1837
- stub_elastic
1838
- driver.run(default_tag: 'test') do
1839
- driver.feed(sample_record)
1840
- end
1841
- assert_nil(index_cmds)
1842
- end
1843
-
1844
- def test_create_should_not_write_if_theres_no_id
1845
- driver.configure("write_operation create\n")
1846
- stub_elastic_ping
1847
- stub_elastic
1848
- driver.run(default_tag: 'test') do
1849
- driver.feed(sample_record)
1850
- end
1851
- assert_nil(index_cmds)
1852
- end
1853
-
1854
- def test_update_should_write_update_op_and_doc_as_upsert_is_false
1855
- driver.configure("write_operation update
1856
- id_key request_id")
1857
- stub_elastic_ping
1858
- stub_elastic
1859
- driver.run(default_tag: 'test') do
1860
- driver.feed(sample_record)
1861
- end
1862
- assert(index_cmds[0].has_key?("update"))
1863
- assert(!index_cmds[1]["doc_as_upsert"])
1864
- assert(!index_cmds[1]["upsert"])
1865
- end
1866
-
1867
- def test_update_should_remove_keys_from_doc_when_keys_are_skipped
1868
- driver.configure("write_operation update
1869
- id_key request_id
1870
- remove_keys_on_update parent_id")
1871
- stub_elastic_ping
1872
- stub_elastic
1873
- driver.run(default_tag: 'test') do
1874
- driver.feed(sample_record)
1875
- end
1876
- assert(index_cmds[1]["doc"])
1877
- assert(!index_cmds[1]["doc"]["parent_id"])
1878
- end
1879
-
1880
- def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
1881
- driver.configure("write_operation upsert
1882
- id_key request_id")
1883
- stub_elastic_ping
1884
- stub_elastic
1885
- driver.run(default_tag: 'test') do
1886
- driver.feed(sample_record)
1887
- end
1888
- assert(index_cmds[0].has_key?("update"))
1889
- assert(index_cmds[1]["doc_as_upsert"])
1890
- assert(!index_cmds[1]["upsert"])
1891
- end
1892
-
1893
- def test_upsert_should_write_update_op_upsert_and_doc_when_keys_are_skipped
1894
- driver.configure("write_operation upsert
1895
- id_key request_id
1896
- remove_keys_on_update parent_id")
1897
- stub_elastic_ping
1898
- stub_elastic
1899
- driver.run(default_tag: 'test') do
1900
- driver.feed(sample_record)
1901
- end
1902
- assert(index_cmds[0].has_key?("update"))
1903
- assert(!index_cmds[1]["doc_as_upsert"])
1904
- assert(index_cmds[1]["upsert"])
1905
- assert(index_cmds[1]["doc"])
1906
- end
1907
-
1908
- def test_upsert_should_remove_keys_from_doc_when_keys_are_skipped
1909
- driver.configure("write_operation upsert
1910
- id_key request_id
1911
- remove_keys_on_update parent_id")
1912
- stub_elastic_ping
1913
- stub_elastic
1914
- driver.run(default_tag: 'test') do
1915
- driver.feed(sample_record)
1916
- end
1917
- assert(index_cmds[1]["upsert"] != index_cmds[1]["doc"])
1918
- assert(!index_cmds[1]["doc"]["parent_id"])
1919
- assert(index_cmds[1]["upsert"]["parent_id"])
1920
- end
1921
-
1922
- def test_upsert_should_remove_multiple_keys_when_keys_are_skipped
1923
- driver.configure("write_operation upsert
1924
- id_key id
1925
- remove_keys_on_update foo,baz")
1926
- stub_elastic_ping
1927
- stub_elastic
1928
- driver.run(default_tag: 'test') do
1929
- driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
1930
- end
1931
- assert(
1932
- index_cmds[1]["doc"] == {
1933
- "id" => 1,
1934
- "zip" => "zam",
1935
- }
1936
- )
1937
- assert(
1938
- index_cmds[1]["upsert"] == {
1939
- "id" => 1,
1940
- "foo" => "bar",
1941
- "baz" => "quix",
1942
- "zip" => "zam",
1943
- }
1944
- )
1945
- end
1946
-
1947
- def test_upsert_should_remove_keys_from_when_the_keys_are_in_the_record
1948
- driver.configure("write_operation upsert
1949
- id_key id
1950
- remove_keys_on_update_key keys_to_skip")
1951
- stub_elastic_ping
1952
- stub_elastic
1953
- driver.run(default_tag: 'test') do
1954
- driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
1955
- end
1956
- assert(
1957
- index_cmds[1]["doc"] == {
1958
- "id" => 1,
1959
- "foo" => "bar",
1960
- }
1961
- )
1962
- assert(
1963
- index_cmds[1]["upsert"] == {
1964
- "id" => 1,
1965
- "foo" => "bar",
1966
- "baz" => "quix",
1967
- }
1968
- )
1969
- end
1970
-
1971
- def test_upsert_should_remove_keys_from_key_on_record_has_higher_presedence_than_config
1972
- driver.configure("write_operation upsert
1973
- id_key id
1974
- remove_keys_on_update foo,bar
1975
- remove_keys_on_update_key keys_to_skip")
1976
- stub_elastic_ping
1977
- stub_elastic
1978
- driver.run(default_tag: 'test') do
1979
- driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
1980
- end
1981
- assert(
1982
- index_cmds[1]["doc"] == {
1983
- "id" => 1,
1984
- # we only expect baz to be stripped here, if the config was more important
1985
- # foo would be stripped too.
1986
- "foo" => "bar",
1987
- }
1988
- )
1989
- assert(
1990
- index_cmds[1]["upsert"] == {
1991
- "id" => 1,
1992
- "foo" => "bar",
1993
- "baz" => "quix",
1994
- }
1995
- )
1996
- end
1997
-
1998
- def test_create_should_write_create_op
1999
- driver.configure("write_operation create
2000
- id_key request_id")
2001
- stub_elastic_ping
2002
- stub_elastic
2003
- driver.run(default_tag: 'test') do
2004
- driver.feed(sample_record)
2005
- end
2006
- assert(index_cmds[0].has_key?("create"))
2007
- end
2008
-
2009
- end
1
+ require 'helper'
2
+ require 'date'
3
+ require 'fluent/test/helpers'
4
+ require 'json'
5
+ require 'fluent/test/driver/output'
6
+ require 'flexmock/test_unit'
7
+
8
+ class ElasticsearchOutput < Test::Unit::TestCase
9
+ include FlexMock::TestCase
10
+ include Fluent::Test::Helpers
11
+
12
+ attr_accessor :index_cmds, :index_command_counts
13
+
14
+ def setup
15
+ Fluent::Test.setup
16
+ require 'fluent/plugin/out_elasticsearch'
17
+ @driver = nil
18
+ log = Fluent::Engine.log
19
+ log.out.logs.slice!(0, log.out.logs.length)
20
+ end
21
+
22
+ def driver(conf='', es_version=5)
23
+ # For request stub to detect compatibility.
24
+ @es_version ||= es_version
25
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
26
+ def detect_es_major_version
27
+ #{@es_version}
28
+ end
29
+ CODE
30
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
31
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
32
+ if !defined?(Fluent::Plugin::Output)
33
+ def format(tag, time, record)
34
+ [time, record].to_msgpack
35
+ end
36
+ end
37
+ }.configure(conf)
38
+ end
39
+
40
+ def default_type_name
41
+ Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
42
+ end
43
+
44
+ def sample_record
45
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
46
+ end
47
+
48
+ def nested_sample_record
49
+ {'nested' =>
50
+ {'age' => 26, 'parent_id' => 'parent', 'routing_id' => 'routing', 'request_id' => '42'}
51
+ }
52
+ end
53
+
54
+ def stub_elastic_ping(url="http://localhost:9200")
55
+ stub_request(:head, url).to_return(:status => 200, :body => "", :headers => {})
56
+ end
57
+
58
+ def stub_elastic(url="http://localhost:9200/_bulk")
59
+ stub_request(:post, url).with do |req|
60
+ @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
61
+ end
62
+ end
63
+
64
+ def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
65
+ stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
66
+ end
67
+
68
+ def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
69
+ if @index_command_counts == nil
70
+ @index_command_counts = {}
71
+ @index_command_counts.default = 0
72
+ end
73
+
74
+ stub_request(:post, url).with do |req|
75
+ index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
76
+ @index_command_counts[url] += index_cmds.size
77
+ end
78
+ end
79
+
80
+ def make_response_body(req, error_el = nil, error_status = nil, error = nil)
81
+ req_index_cmds = req.body.split("\n").map { |r| JSON.parse(r) }
82
+ items = []
83
+ count = 0
84
+ ids = 1
85
+ op = nil
86
+ index = nil
87
+ type = nil
88
+ id = nil
89
+ req_index_cmds.each do |cmd|
90
+ if count.even?
91
+ op = cmd.keys[0]
92
+ index = cmd[op]['_index']
93
+ type = cmd[op]['_type']
94
+ if cmd[op].has_key?('_id')
95
+ id = cmd[op]['_id']
96
+ else
97
+ # Note: this appears to be an undocumented feature of Elasticsearch
98
+ # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/docs-bulk.html
99
+ # When you submit an "index" write_operation, with no "_id" field in the
100
+ # metadata header, Elasticsearch will turn this into a "create"
101
+ # operation in the response.
102
+ if "index" == op
103
+ op = "create"
104
+ end
105
+ id = ids
106
+ ids += 1
107
+ end
108
+ else
109
+ item = {
110
+ op => {
111
+ '_index' => index, '_type' => type, '_id' => id, '_version' => 1,
112
+ '_shards' => { 'total' => 1, 'successful' => 1, 'failed' => 0 },
113
+ 'status' => op == 'create' ? 201 : 200
114
+ }
115
+ }
116
+ items.push(item)
117
+ end
118
+ count += 1
119
+ end
120
+ if !error_el.nil? && !error_status.nil? && !error.nil?
121
+ op = items[error_el].keys[0]
122
+ items[error_el][op].delete('_version')
123
+ items[error_el][op].delete('_shards')
124
+ items[error_el][op]['error'] = error
125
+ items[error_el][op]['status'] = error_status
126
+ errors = true
127
+ else
128
+ errors = false
129
+ end
130
+ @index_cmds = items
131
+ body = { 'took' => 6, 'errors' => errors, 'items' => items }
132
+ return body.to_json
133
+ end
134
+
135
+ def stub_elastic_bad_argument(url="http://localhost:9200/_bulk")
136
+ error = {
137
+ "type" => "mapper_parsing_exception",
138
+ "reason" => "failed to parse [...]",
139
+ "caused_by" => {
140
+ "type" => "illegal_argument_exception",
141
+ "reason" => "Invalid format: \"...\""
142
+ }
143
+ }
144
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 400, error), :headers => { 'Content-Type' => 'json' } } })
145
+ end
146
+
147
+ def stub_elastic_bulk_error(url="http://localhost:9200/_bulk")
148
+ error = {
149
+ "type" => "some-unrecognized-error",
150
+ "reason" => "some message printed here ...",
151
+ }
152
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
153
+ end
154
+
155
+ def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
156
+ error = {
157
+ "status" => 500,
158
+ "type" => "es_rejected_execution_exception",
159
+ "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
160
+ }
161
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 429, error), :headers => { 'Content-Type' => 'json' } } })
162
+ end
163
+
164
+ def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
165
+ error = {
166
+ "status" => 500,
167
+ "type" => "out_of_memory_error",
168
+ "reason" => "Java heap space"
169
+ }
170
+ stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
171
+ end
172
+
173
+ def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
174
+ error = {
175
+ "category" => "some-other-type",
176
+ "reason" => "some-other-reason"
177
+ }
178
+ stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
179
+ end
180
+
181
+ def assert_logs_include(logs, msg)
182
+ matches = logs.grep /#{msg}/
183
+ assert_equal(1, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
184
+ end
185
+
186
+ def test_configure
187
+ config = %{
188
+ host logs.google.com
189
+ port 777
190
+ scheme https
191
+ path /es/
192
+ user john
193
+ password doe
194
+ }
195
+ instance = driver(config).instance
196
+
197
+ assert_equal 'logs.google.com', instance.host
198
+ assert_equal 777, instance.port
199
+ assert_equal 'https', instance.scheme
200
+ assert_equal '/es/', instance.path
201
+ assert_equal 'john', instance.user
202
+ assert_equal 'doe', instance.password
203
+ assert_equal :TLSv1, instance.ssl_version
204
+ assert_nil instance.client_key
205
+ assert_nil instance.client_cert
206
+ assert_nil instance.client_key_pass
207
+ assert_false instance.with_transporter_log
208
+ assert_equal :"application/json", instance.content_type
209
+ assert_equal "fluentd", default_type_name
210
+ end
211
+
212
+ test 'configure Content-Type' do
213
+ config = %{
214
+ content_type application/x-ndjson
215
+ }
216
+ instance = driver(config).instance
217
+ assert_equal :"application/x-ndjson", instance.content_type
218
+ end
219
+
220
+ test 'invalid Content-Type' do
221
+ config = %{
222
+ content_type nonexistent/invalid
223
+ }
224
+ assert_raise(Fluent::ConfigError) {
225
+ instance = driver(config).instance
226
+ }
227
+ end
228
+
229
+ test 'Detected Elasticsearch 7' do
230
+ config = %{
231
+ type_name changed
232
+ }
233
+ instance = driver(config, 7).instance
234
+ assert_equal '_doc', instance.type_name
235
+ end
236
+
237
+ test 'lack of tag in chunk_keys' do
238
+ assert_raise_message(/'tag' in chunk_keys is required./) do
239
+ driver(Fluent::Config::Element.new(
240
+ 'ROOT', '', {
241
+ '@type' => 'elasticsearch',
242
+ 'host' => 'log.google.com',
243
+ 'port' => 777,
244
+ 'scheme' => 'https',
245
+ 'path' => '/es/',
246
+ 'user' => 'john',
247
+ 'pasword' => 'doe',
248
+ }, [
249
+ Fluent::Config::Element.new('buffer', 'mykey', {
250
+ 'chunk_keys' => 'mykey'
251
+ }, [])
252
+ ]
253
+ ))
254
+ end
255
+ end
256
+
257
+ def test_template_already_present
258
+ config = %{
259
+ host logs.google.com
260
+ port 777
261
+ scheme https
262
+ path /es/
263
+ user john
264
+ password doe
265
+ template_name logstash
266
+ template_file /abc123
267
+ }
268
+
269
+ # connection start
270
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
271
+ to_return(:status => 200, :body => "", :headers => {})
272
+ # check if template exists
273
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
274
+ to_return(:status => 200, :body => "", :headers => {})
275
+
276
+ driver(config)
277
+
278
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
279
+ end
280
+
281
+ def test_template_create
282
+ cwd = File.dirname(__FILE__)
283
+ template_file = File.join(cwd, 'test_template.json')
284
+
285
+ config = %{
286
+ host logs.google.com
287
+ port 777
288
+ scheme https
289
+ path /es/
290
+ user john
291
+ password doe
292
+ template_name logstash
293
+ template_file #{template_file}
294
+ }
295
+
296
+ # connection start
297
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
298
+ to_return(:status => 200, :body => "", :headers => {})
299
+ # check if template exists
300
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
301
+ to_return(:status => 404, :body => "", :headers => {})
302
+ # creation
303
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
304
+ to_return(:status => 200, :body => "", :headers => {})
305
+
306
+ driver(config)
307
+
308
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
309
+ end
310
+
311
+ def test_template_overwrite
312
+ cwd = File.dirname(__FILE__)
313
+ template_file = File.join(cwd, 'test_template.json')
314
+
315
+ config = %{
316
+ host logs.google.com
317
+ port 777
318
+ scheme https
319
+ path /es/
320
+ user john
321
+ password doe
322
+ template_name logstash
323
+ template_file #{template_file}
324
+ template_overwrite true
325
+ }
326
+
327
+ # connection start
328
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
329
+ to_return(:status => 200, :body => "", :headers => {})
330
+ # check if template exists
331
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
332
+ to_return(:status => 200, :body => "", :headers => {})
333
+ # creation
334
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
335
+ to_return(:status => 200, :body => "", :headers => {})
336
+
337
+ driver(config)
338
+
339
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
340
+ end
341
+
342
+
343
+ def test_template_create_invalid_filename
344
+ config = %{
345
+ host logs.google.com
346
+ port 777
347
+ scheme https
348
+ path /es/
349
+ user john
350
+ password doe
351
+ template_name logstash
352
+ template_file /abc123
353
+ }
354
+
355
+ # connection start
356
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
357
+ to_return(:status => 200, :body => "", :headers => {})
358
+ # check if template exists
359
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
360
+ to_return(:status => 404, :body => "", :headers => {})
361
+
362
+ assert_raise(RuntimeError) {
363
+ driver(config)
364
+ }
365
+ end
366
+
367
+ def test_template_retry_install
368
+ cwd = File.dirname(__FILE__)
369
+ template_file = File.join(cwd, 'test_template.json')
370
+
371
+ config = %{
372
+ host logs.google.com
373
+ port 778
374
+ scheme https
375
+ path /es/
376
+ user john
377
+ password doe
378
+ template_name logstash
379
+ template_file #{template_file}
380
+ max_retry_putting_template 3
381
+ }
382
+
383
+ connection_resets = 0
384
+ # connection start
385
+ stub_request(:head, "https://john:doe@logs.google.com:778/es//").with do |req|
386
+ connection_resets += 1
387
+ end
388
+ # check if template exists
389
+ stub_request(:get, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
390
+ raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
391
+ end
392
+ # creation
393
+ stub_request(:put, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
394
+ raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
395
+ end
396
+
397
+ assert_raise(Fluent::Plugin::ElasticsearchOutput::ConnectionFailure) do
398
+ driver(config)
399
+ end
400
+
401
+ assert_equal(connection_resets, 4)
402
+ end
403
+
404
+ def test_templates_create
405
+ cwd = File.dirname(__FILE__)
406
+ template_file = File.join(cwd, 'test_template.json')
407
+ config = %{
408
+ host logs.google.com
409
+ port 777
410
+ scheme https
411
+ path /es/
412
+ user john
413
+ password doe
414
+ templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
415
+ }
416
+
417
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
418
+ to_return(:status => 200, :body => "", :headers => {})
419
+ # check if template exists
420
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
421
+ to_return(:status => 404, :body => "", :headers => {})
422
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
423
+ to_return(:status => 404, :body => "", :headers => {})
424
+
425
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
426
+ to_return(:status => 200, :body => "", :headers => {}) #exists
427
+
428
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
429
+ to_return(:status => 200, :body => "", :headers => {})
430
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
431
+ to_return(:status => 200, :body => "", :headers => {})
432
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
433
+ to_return(:status => 200, :body => "", :headers => {})
434
+
435
+ driver(config)
436
+
437
+ assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
438
+ assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
439
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
440
+ end
441
+
442
+ def test_templates_overwrite
443
+ cwd = File.dirname(__FILE__)
444
+ template_file = File.join(cwd, 'test_template.json')
445
+ config = %{
446
+ host logs.google.com
447
+ port 777
448
+ scheme https
449
+ path /es/
450
+ user john
451
+ password doe
452
+ templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
453
+ template_overwrite true
454
+ }
455
+
456
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
457
+ to_return(:status => 200, :body => "", :headers => {})
458
+ # check if template exists
459
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
460
+ to_return(:status => 200, :body => "", :headers => {})
461
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
462
+ to_return(:status => 200, :body => "", :headers => {})
463
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
464
+ to_return(:status => 200, :body => "", :headers => {}) #exists
465
+
466
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
467
+ to_return(:status => 200, :body => "", :headers => {})
468
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
469
+ to_return(:status => 200, :body => "", :headers => {})
470
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
471
+ to_return(:status => 200, :body => "", :headers => {})
472
+
473
+ driver(config)
474
+
475
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
476
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
477
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
478
+ end
479
+
480
+ def test_templates_not_used
481
+ cwd = File.dirname(__FILE__)
482
+ template_file = File.join(cwd, 'test_template.json')
483
+
484
+ config = %{
485
+ host logs.google.com
486
+ port 777
487
+ scheme https
488
+ path /es/
489
+ user john
490
+ password doe
491
+ template_name logstash
492
+ template_file #{template_file}
493
+ templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
494
+ }
495
+ # connection start
496
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
497
+ to_return(:status => 200, :body => "", :headers => {})
498
+ # check if template exists
499
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
500
+ to_return(:status => 404, :body => "", :headers => {})
501
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
502
+ to_return(:status => 404, :body => "", :headers => {})
503
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
504
+ to_return(:status => 404, :body => "", :headers => {})
505
+ #creation
506
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
507
+ to_return(:status => 200, :body => "", :headers => {})
508
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
509
+ to_return(:status => 200, :body => "", :headers => {})
510
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
511
+ to_return(:status => 200, :body => "", :headers => {})
512
+
513
+ driver(config)
514
+
515
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
516
+
517
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1")
518
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
519
+ end
520
+
521
+ def test_templates_can_be_partially_created_if_error_occurs
522
+ cwd = File.dirname(__FILE__)
523
+ template_file = File.join(cwd, 'test_template.json')
524
+ config = %{
525
+ host logs.google.com
526
+ port 777
527
+ scheme https
528
+ path /es/
529
+ user john
530
+ password doe
531
+ templates {"logstash1":"#{template_file}", "logstash2":"/abc" }
532
+ }
533
+ stub_request(:head, "https://john:doe@logs.google.com:777/es//").
534
+ to_return(:status => 200, :body => "", :headers => {})
535
+ # check if template exists
536
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
537
+ to_return(:status => 404, :body => "", :headers => {})
538
+ stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
539
+ to_return(:status => 404, :body => "", :headers => {})
540
+
541
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
542
+ to_return(:status => 200, :body => "", :headers => {})
543
+ stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
544
+ to_return(:status => 200, :body => "", :headers => {})
545
+
546
+ assert_raise(RuntimeError) {
547
+ driver(config)
548
+ }
549
+
550
+ assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
551
+ assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2")
552
+ end
553
+
554
+ def test_legacy_hosts_list
555
+ config = %{
556
+ hosts host1:50,host2:100,host3
557
+ scheme https
558
+ path /es/
559
+ port 123
560
+ }
561
+ instance = driver(config).instance
562
+
563
+ assert_equal 3, instance.get_connection_options[:hosts].length
564
+ host1, host2, host3 = instance.get_connection_options[:hosts]
565
+
566
+ assert_equal 'host1', host1[:host]
567
+ assert_equal 50, host1[:port]
568
+ assert_equal 'https', host1[:scheme]
569
+ assert_equal '/es/', host2[:path]
570
+ assert_equal 'host3', host3[:host]
571
+ assert_equal 123, host3[:port]
572
+ assert_equal 'https', host3[:scheme]
573
+ assert_equal '/es/', host3[:path]
574
+ end
575
+
576
+ def test_hosts_list
577
+ config = %{
578
+ hosts https://john:password@host1:443/elastic/,http://host2
579
+ path /default_path
580
+ user default_user
581
+ password default_password
582
+ }
583
+ instance = driver(config).instance
584
+
585
+ assert_equal 2, instance.get_connection_options[:hosts].length
586
+ host1, host2 = instance.get_connection_options[:hosts]
587
+
588
+ assert_equal 'host1', host1[:host]
589
+ assert_equal 443, host1[:port]
590
+ assert_equal 'https', host1[:scheme]
591
+ assert_equal 'john', host1[:user]
592
+ assert_equal 'password', host1[:password]
593
+ assert_equal '/elastic/', host1[:path]
594
+
595
+ assert_equal 'host2', host2[:host]
596
+ assert_equal 'http', host2[:scheme]
597
+ assert_equal 'default_user', host2[:user]
598
+ assert_equal 'default_password', host2[:password]
599
+ assert_equal '/default_path', host2[:path]
600
+ end
601
+
602
+ def test_hosts_list_with_escape_placeholders
603
+ config = %{
604
+ hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
605
+ path /default_path
606
+ user default_user
607
+ password default_password
608
+ }
609
+ instance = driver(config).instance
610
+
611
+ assert_equal 2, instance.get_connection_options[:hosts].length
612
+ host1, host2 = instance.get_connection_options[:hosts]
613
+
614
+ assert_equal 'host1', host1[:host]
615
+ assert_equal 443, host1[:port]
616
+ assert_equal 'https', host1[:scheme]
617
+ assert_equal 'j%2Bhn', host1[:user]
618
+ assert_equal 'passw%40rd', host1[:password]
619
+ assert_equal '/elastic/', host1[:path]
620
+
621
+ assert_equal 'host2', host2[:host]
622
+ assert_equal 'http', host2[:scheme]
623
+ assert_equal 'default_user', host2[:user]
624
+ assert_equal 'default_password', host2[:password]
625
+ assert_equal '/default_path', host2[:path]
626
+ end
627
+
628
+ def test_single_host_params_and_defaults
629
+ config = %{
630
+ host logs.google.com
631
+ user john
632
+ password doe
633
+ }
634
+ instance = driver(config).instance
635
+
636
+ assert_equal 1, instance.get_connection_options[:hosts].length
637
+ host1 = instance.get_connection_options[:hosts][0]
638
+
639
+ assert_equal 'logs.google.com', host1[:host]
640
+ assert_equal 9200, host1[:port]
641
+ assert_equal 'http', host1[:scheme]
642
+ assert_equal 'john', host1[:user]
643
+ assert_equal 'doe', host1[:password]
644
+ assert_equal nil, host1[:path]
645
+ end
646
+
647
+ def test_single_host_params_and_defaults_with_escape_placeholders
648
+ config = %{
649
+ host logs.google.com
650
+ user %{j+hn}
651
+ password %{d@e}
652
+ }
653
+ instance = driver(config).instance
654
+
655
+ assert_equal 1, instance.get_connection_options[:hosts].length
656
+ host1 = instance.get_connection_options[:hosts][0]
657
+
658
+ assert_equal 'logs.google.com', host1[:host]
659
+ assert_equal 9200, host1[:port]
660
+ assert_equal 'http', host1[:scheme]
661
+ assert_equal 'j%2Bhn', host1[:user]
662
+ assert_equal 'd%40e', host1[:password]
663
+ assert_equal nil, host1[:path]
664
+ end
665
+
666
+ def test_content_type_header
667
+ stub_request(:head, "http://localhost:9200/").
668
+ to_return(:status => 200, :body => "", :headers => {})
669
+ if Elasticsearch::VERSION >= "6.0.2"
670
+ elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
671
+ with(headers: { "Content-Type" => "application/x-ndjson" })
672
+ else
673
+ elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
674
+ with(headers: { "Content-Type" => "application/json" })
675
+ end
676
+ driver.run(default_tag: 'test') do
677
+ driver.feed(sample_record)
678
+ end
679
+ assert_requested(elastic_request)
680
+ end
681
+
682
+ def test_write_message_with_bad_chunk
683
+ driver.configure("target_index_key bad_value\n@log_level debug\n")
684
+ stub_elastic_ping
685
+ stub_elastic
686
+ driver.run(default_tag: 'test') do
687
+ driver.feed({'bad_value'=>"\255"})
688
+ end
689
+ error_log = driver.error_events.map {|e| e.last.message }
690
+
691
+ assert_logs_include(error_log, /(input string invalid)|(invalid byte sequence in UTF-8)/)
692
+ end
693
+
694
+ def test_writes_to_default_index
695
+ stub_elastic_ping
696
+ stub_elastic
697
+ driver.run(default_tag: 'test') do
698
+ driver.feed(sample_record)
699
+ end
700
+ assert_equal('fluentd', index_cmds.first['index']['_index'])
701
+ end
702
+
703
+ def test_writes_to_default_type
704
+ stub_elastic_ping
705
+ stub_elastic
706
+ driver.run(default_tag: 'test') do
707
+ driver.feed(sample_record)
708
+ end
709
+ assert_equal(default_type_name, index_cmds.first['index']['_type'])
710
+ end
711
+
712
+ def test_writes_to_speficied_index
713
+ driver.configure("index_name myindex\n")
714
+ stub_elastic_ping
715
+ stub_elastic
716
+ driver.run(default_tag: 'test') do
717
+ driver.feed(sample_record)
718
+ end
719
+ assert_equal('myindex', index_cmds.first['index']['_index'])
720
+ end
721
+
722
+ class IndexNamePlaceholdersTest < self
723
+ def test_writes_to_speficied_index_with_tag_placeholder
724
+ driver.configure("index_name myindex.${tag}\n")
725
+ stub_elastic_ping
726
+ stub_elastic
727
+ driver.run(default_tag: 'test') do
728
+ driver.feed(sample_record)
729
+ end
730
+ assert_equal('myindex.test', index_cmds.first['index']['_index'])
731
+ end
732
+
733
+ def test_writes_to_speficied_index_with_time_placeholder
734
+ driver.configure(Fluent::Config::Element.new(
735
+ 'ROOT', '', {
736
+ '@type' => 'elasticsearch',
737
+ 'index_name' => 'myindex.%Y.%m.%d',
738
+ }, [
739
+ Fluent::Config::Element.new('buffer', 'tag,time', {
740
+ 'chunk_keys' => ['tag', 'time'],
741
+ 'timekey' => 3600,
742
+ }, [])
743
+ ]
744
+ ))
745
+ stub_elastic_ping
746
+ stub_elastic
747
+ time = Time.parse Date.today.iso8601
748
+ driver.run(default_tag: 'test') do
749
+ driver.feed(time.to_i, sample_record)
750
+ end
751
+ assert_equal("myindex.#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['index']['_index'])
752
+ end
753
+
754
+ def test_writes_to_speficied_index_with_custom_key_placeholder
755
+ driver.configure(Fluent::Config::Element.new(
756
+ 'ROOT', '', {
757
+ '@type' => 'elasticsearch',
758
+ 'index_name' => 'myindex.${pipeline_id}',
759
+ }, [
760
+ Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
761
+ ]
762
+ ))
763
+ time = Time.parse Date.today.iso8601
764
+ pipeline_id = "mypipeline"
765
+ logstash_index = "myindex.#{pipeline_id}"
766
+ stub_elastic_ping
767
+ stub_elastic
768
+ driver.run(default_tag: 'test') do
769
+ driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
770
+ end
771
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
772
+ end
773
+ end
774
+
775
+ def test_writes_to_speficied_index_uppercase
776
+ driver.configure("index_name MyIndex\n")
777
+ stub_elastic_ping
778
+ stub_elastic
779
+ driver.run(default_tag: 'test') do
780
+ driver.feed(sample_record)
781
+ end
782
+ # Allthough index_name has upper-case characters,
783
+ # it should be set as lower-case when sent to elasticsearch.
784
+ assert_equal('myindex', index_cmds.first['index']['_index'])
785
+ end
786
+
787
+ def test_writes_to_target_index_key
788
+ driver.configure("target_index_key @target_index\n")
789
+ stub_elastic_ping
790
+ stub_elastic
791
+ record = sample_record.clone
792
+ driver.run(default_tag: 'test') do
793
+ driver.feed(sample_record.merge('@target_index' => 'local-override'))
794
+ end
795
+ assert_equal('local-override', index_cmds.first['index']['_index'])
796
+ assert_nil(index_cmds[1]['@target_index'])
797
+ end
798
+
799
+ def test_writes_to_target_index_key_logstash
800
+ driver.configure("target_index_key @target_index
801
+ logstash_format true")
802
+ time = Time.parse Date.today.iso8601
803
+ stub_elastic_ping
804
+ stub_elastic
805
+ driver.run(default_tag: 'test') do
806
+ driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
807
+ end
808
+ assert_equal('local-override', index_cmds.first['index']['_index'])
809
+ end
810
+
811
+ def test_writes_to_target_index_key_logstash_uppercase
812
+ driver.configure("target_index_key @target_index
813
+ logstash_format true")
814
+ time = Time.parse Date.today.iso8601
815
+ stub_elastic_ping
816
+ stub_elastic
817
+ driver.run(default_tag: 'test') do
818
+ driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
819
+ end
820
+ # Allthough @target_index has upper-case characters,
821
+ # it should be set as lower-case when sent to elasticsearch.
822
+ assert_equal('local-override', index_cmds.first['index']['_index'])
823
+ end
824
+
825
+ def test_writes_to_default_index_with_pipeline
826
+ pipeline = "fluentd"
827
+ driver.configure("pipeline #{pipeline}")
828
+ stub_elastic_ping
829
+ stub_elastic
830
+ driver.run(default_tag: 'test') do
831
+ driver.feed(sample_record)
832
+ end
833
+ assert_equal(pipeline, index_cmds.first['index']['pipeline'])
834
+ end
835
+
836
+ def test_writes_to_target_index_key_fallack
837
+ driver.configure("target_index_key @target_index\n")
838
+ stub_elastic_ping
839
+ stub_elastic
840
+ driver.run(default_tag: 'test') do
841
+ driver.feed(sample_record)
842
+ end
843
+ assert_equal('fluentd', index_cmds.first['index']['_index'])
844
+ end
845
+
846
+ def test_writes_to_target_index_key_fallack_logstash
847
+ driver.configure("target_index_key @target_index\n
848
+ logstash_format true")
849
+ time = Time.parse Date.today.iso8601
850
+ logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
851
+ stub_elastic_ping
852
+ stub_elastic
853
+ driver.run(default_tag: 'test') do
854
+ driver.feed(time.to_i, sample_record)
855
+ end
856
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
857
+ end
858
+
859
+ data("border" => {"es_version" => 6, "_type" => "mytype"},
860
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
861
+ )
862
+ def test_writes_to_speficied_type(data)
863
+ driver('', data["es_version"]).configure("type_name mytype\n")
864
+ stub_elastic_ping
865
+ stub_elastic
866
+ driver.run(default_tag: 'test') do
867
+ driver.feed(sample_record)
868
+ end
869
+ assert_equal(data['_type'], index_cmds.first['index']['_type'])
870
+ end
871
+
872
+ data("border" => {"es_version" => 6, "_type" => "mytype.test"},
873
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
874
+ )
875
+ def test_writes_to_speficied_type_with_placeholders(data)
876
+ driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
877
+ stub_elastic_ping
878
+ stub_elastic
879
+ driver.run(default_tag: 'test') do
880
+ driver.feed(sample_record)
881
+ end
882
+ assert_equal(data['_type'], index_cmds.first['index']['_type'])
883
+ end
884
+
885
+ data("old" => {"es_version" => 2, "_type" => "local-override"},
886
+ "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
887
+ "border" => {"es_version" => 6, "_type" => "fluentd"},
888
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
889
+ )
890
+ def test_writes_to_target_type_key(data)
891
+ driver('', data["es_version"]).configure("target_type_key @target_type\n")
892
+ stub_elastic_ping
893
+ stub_elastic
894
+ record = sample_record.clone
895
+ driver.run(default_tag: 'test') do
896
+ driver.feed(sample_record.merge('@target_type' => 'local-override'))
897
+ end
898
+ assert_equal(data["_type"], index_cmds.first['index']['_type'])
899
+ assert_nil(index_cmds[1]['@target_type'])
900
+ end
901
+
902
+ def test_writes_to_target_type_key_fallack_to_default
903
+ driver.configure("target_type_key @target_type\n")
904
+ stub_elastic_ping
905
+ stub_elastic
906
+ driver.run(default_tag: 'test') do
907
+ driver.feed(sample_record)
908
+ end
909
+ assert_equal(default_type_name, index_cmds.first['index']['_type'])
910
+ end
911
+
912
+ def test_writes_to_target_type_key_fallack_to_type_name
913
+ driver.configure("target_type_key @target_type
914
+ type_name mytype")
915
+ stub_elastic_ping
916
+ stub_elastic
917
+ driver.run(default_tag: 'test') do
918
+ driver.feed(sample_record)
919
+ end
920
+ assert_equal('mytype', index_cmds.first['index']['_type'])
921
+ end
922
+
923
+ data("old" => {"es_version" => 2, "_type" => "local-override"},
924
+ "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
925
+ "border" => {"es_version" => 6, "_type" => "fluentd"},
926
+ "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
927
+ )
928
+ def test_writes_to_target_type_key_nested(data)
929
+ driver('', data["es_version"]).configure("target_type_key kubernetes.labels.log_type\n")
930
+ stub_elastic_ping
931
+ stub_elastic
932
+ driver.run(default_tag: 'test') do
933
+ driver.feed(sample_record.merge('kubernetes' => {
934
+ 'labels' => {
935
+ 'log_type' => 'local-override'
936
+ }
937
+ }))
938
+ end
939
+ assert_equal(data["_type"], index_cmds.first['index']['_type'])
940
+ assert_nil(index_cmds[1]['kubernetes']['labels']['log_type'])
941
+ end
942
+
943
+ def test_writes_to_target_type_key_fallack_to_default_nested
944
+ driver.configure("target_type_key kubernetes.labels.log_type\n")
945
+ stub_elastic_ping
946
+ stub_elastic
947
+ driver.run(default_tag: 'test') do
948
+ driver.feed(sample_record.merge('kubernetes' => {
949
+ 'labels' => {
950
+ 'other_labels' => 'test'
951
+ }
952
+ }))
953
+ end
954
+ assert_equal(default_type_name, index_cmds.first['index']['_type'])
955
+ end
956
+
957
+ def test_writes_to_speficied_host
958
+ driver.configure("host 192.168.33.50\n")
959
+ stub_elastic_ping("http://192.168.33.50:9200")
960
+ elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
961
+ driver.run(default_tag: 'test') do
962
+ driver.feed(sample_record)
963
+ end
964
+ assert_requested(elastic_request)
965
+ end
966
+
967
+ def test_writes_to_speficied_port
968
+ driver.configure("port 9201\n")
969
+ stub_elastic_ping("http://localhost:9201")
970
+ elastic_request = stub_elastic("http://localhost:9201/_bulk")
971
+ driver.run(default_tag: 'test') do
972
+ driver.feed(sample_record)
973
+ end
974
+ assert_requested(elastic_request)
975
+ end
976
+
977
+ def test_writes_to_multi_hosts
978
+ hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
979
+ hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
980
+
981
+ driver.configure("hosts #{hosts_string}")
982
+
983
+ hosts.each do |host_info|
984
+ host, port = host_info
985
+ stub_elastic_ping("http://#{host}:#{port}")
986
+ stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
987
+ end
988
+
989
+ driver.run(default_tag: 'test') do
990
+ 1000.times do
991
+ driver.feed(sample_record.merge('age'=>rand(100)))
992
+ end
993
+ end
994
+
995
+ # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
996
+ # it's Fluentd test driver's constraint
997
+ # so @index_command_counts.size is always 1
998
+
999
+ assert(@index_command_counts.size > 0, "not working with hosts options")
1000
+
1001
+ total = 0
1002
+ @index_command_counts.each do |url, count|
1003
+ total += count
1004
+ end
1005
+ assert_equal(2000, total)
1006
+ end
1007
+
1008
+ def test_nested_record_with_flattening_on
1009
+ driver.configure("flatten_hashes true
1010
+ flatten_hashes_separator |")
1011
+
1012
+ original_hash = {"foo" => {"bar" => "baz"}, "people" => [
1013
+ {"age" => "25", "height" => "1ft"},
1014
+ {"age" => "30", "height" => "2ft"}
1015
+ ]}
1016
+
1017
+ expected_output = {"foo|bar"=>"baz", "people" => [
1018
+ {"age" => "25", "height" => "1ft"},
1019
+ {"age" => "30", "height" => "2ft"}
1020
+ ]}
1021
+
1022
+ stub_elastic_ping
1023
+ stub_elastic
1024
+ driver.run(default_tag: 'test') do
1025
+ driver.feed(original_hash)
1026
+ end
1027
+ assert_equal expected_output, index_cmds[1]
1028
+ end
1029
+
1030
+ def test_nested_record_with_flattening_off
1031
+ # flattening off by default
1032
+
1033
+ original_hash = {"foo" => {"bar" => "baz"}}
1034
+ expected_output = {"foo" => {"bar" => "baz"}}
1035
+
1036
+ stub_elastic_ping
1037
+ stub_elastic
1038
+ driver.run(default_tag: 'test') do
1039
+ driver.feed(original_hash)
1040
+ end
1041
+ assert_equal expected_output, index_cmds[1]
1042
+ end
1043
+
1044
+ def test_makes_bulk_request
1045
+ stub_elastic_ping
1046
+ stub_elastic
1047
+ driver.run(default_tag: 'test') do
1048
+ driver.feed(sample_record)
1049
+ driver.feed(sample_record.merge('age' => 27))
1050
+ end
1051
+ assert_equal(4, index_cmds.count)
1052
+ end
1053
+
1054
+ def test_all_records_are_preserved_in_bulk
1055
+ stub_elastic_ping
1056
+ stub_elastic
1057
+ driver.run(default_tag: 'test') do
1058
+ driver.feed(sample_record)
1059
+ driver.feed(sample_record.merge('age' => 27))
1060
+ end
1061
+ assert_equal(26, index_cmds[1]['age'])
1062
+ assert_equal(27, index_cmds[3]['age'])
1063
+ end
1064
+
1065
+ def test_writes_to_logstash_index
1066
+ driver.configure("logstash_format true\n")
1067
+ #
1068
+ # This is 1 second past midnight in BST, so the UTC index should be the day before
1069
+ dt = DateTime.new(2015, 6, 1, 0, 0, 1, "+01:00")
1070
+ logstash_index = "logstash-2015.05.31"
1071
+ stub_elastic_ping
1072
+ stub_elastic
1073
+ driver.run(default_tag: 'test') do
1074
+ driver.feed(dt.to_time.to_i, sample_record)
1075
+ end
1076
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1077
+ end
1078
+
1079
+ def test_writes_to_logstash_non_utc_index
1080
+ driver.configure("logstash_format true
1081
+ utc_index false")
1082
+ # When using `utc_index false` the index time will be the local day of
1083
+ # ingestion time
1084
+ time = Date.today.to_time
1085
+ index = "logstash-#{time.strftime("%Y.%m.%d")}"
1086
+ stub_elastic_ping
1087
+ stub_elastic
1088
+ driver.run(default_tag: 'test') do
1089
+ driver.feed(time.to_i, sample_record)
1090
+ end
1091
+ assert_equal(index, index_cmds.first['index']['_index'])
1092
+ end
1093
+
1094
+ def test_writes_to_logstash_index_with_specified_prefix
1095
+ driver.configure("logstash_format true
1096
+ logstash_prefix myprefix")
1097
+ time = Time.parse Date.today.iso8601
1098
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1099
+ stub_elastic_ping
1100
+ stub_elastic
1101
+ driver.run(default_tag: 'test') do
1102
+ driver.feed(time.to_i, sample_record)
1103
+ end
1104
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1105
+ end
1106
+
1107
+ def test_writes_to_logstash_index_with_specified_prefix_and_separator
1108
+ separator = '_'
1109
+ driver.configure("logstash_format true
1110
+ logstash_prefix_separator #{separator}
1111
+ logstash_prefix myprefix")
1112
+ time = Time.parse Date.today.iso8601
1113
+ logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
1114
+ stub_elastic_ping
1115
+ stub_elastic
1116
+ driver.run(default_tag: 'test') do
1117
+ driver.feed(time.to_i, sample_record)
1118
+ end
1119
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1120
+ end
1121
+
1122
+ class LogStashPrefixPlaceholdersTest < self
1123
+ def test_writes_to_logstash_index_with_specified_prefix_and_tag_placeholder
1124
+ driver.configure("logstash_format true
1125
+ logstash_prefix myprefix-${tag}")
1126
+ time = Time.parse Date.today.iso8601
1127
+ logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
1128
+ stub_elastic_ping
1129
+ stub_elastic
1130
+ driver.run(default_tag: 'test') do
1131
+ driver.feed(time.to_i, sample_record)
1132
+ end
1133
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1134
+ end
1135
+
1136
+ def test_writes_to_logstash_index_with_specified_prefix_and_time_placeholder
1137
+ driver.configure(Fluent::Config::Element.new(
1138
+ 'ROOT', '', {
1139
+ '@type' => 'elasticsearch',
1140
+ 'logstash_format' => true,
1141
+ 'logstash_prefix' => 'myprefix-%H',
1142
+ }, [
1143
+ Fluent::Config::Element.new('buffer', 'tag,time', {
1144
+ 'chunk_keys' => ['tag', 'time'],
1145
+ 'timekey' => 3600,
1146
+ }, [])
1147
+ ]
1148
+ ))
1149
+ time = Time.parse Date.today.iso8601
1150
+ logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
1151
+ stub_elastic_ping
1152
+ stub_elastic
1153
+ driver.run(default_tag: 'test') do
1154
+ driver.feed(time.to_i, sample_record)
1155
+ end
1156
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1157
+ end
1158
+
1159
+ def test_writes_to_logstash_index_with_specified_prefix_and_custom_key_placeholder
1160
+ driver.configure(Fluent::Config::Element.new(
1161
+ 'ROOT', '', {
1162
+ '@type' => 'elasticsearch',
1163
+ 'logstash_format' => true,
1164
+ 'logstash_prefix' => 'myprefix-${pipeline_id}',
1165
+ }, [
1166
+ Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
1167
+ ]
1168
+ ))
1169
+ time = Time.parse Date.today.iso8601
1170
+ pipeline_id = "mypipeline"
1171
+ logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
1172
+ stub_elastic_ping
1173
+ stub_elastic
1174
+ driver.run(default_tag: 'test') do
1175
+ driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
1176
+ end
1177
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1178
+ end
1179
+ end
1180
+
1181
+ def test_writes_to_logstash_index_with_specified_prefix_uppercase
1182
+ driver.configure("logstash_format true
1183
+ logstash_prefix MyPrefix")
1184
+ time = Time.parse Date.today.iso8601
1185
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
1186
+ stub_elastic_ping
1187
+ stub_elastic
1188
+ driver.run(default_tag: 'test') do
1189
+ driver.feed(time.to_i, sample_record)
1190
+ end
1191
+ # Allthough logstash_prefix has upper-case characters,
1192
+ # it should be set as lower-case when sent to elasticsearch.
1193
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1194
+ end
1195
+
1196
+ def test_writes_to_logstash_index_with_specified_dateformat
1197
+ driver.configure("logstash_format true
1198
+ logstash_dateformat %Y.%m")
1199
+ time = Time.parse Date.today.iso8601
1200
+ logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
1201
+ stub_elastic_ping
1202
+ stub_elastic
1203
+ driver.run(default_tag: 'test') do
1204
+ driver.feed(time.to_i, sample_record)
1205
+ end
1206
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1207
+ end
1208
+
1209
+ def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
1210
+ driver.configure("logstash_format true
1211
+ logstash_prefix myprefix
1212
+ logstash_dateformat %Y.%m")
1213
+ time = Time.parse Date.today.iso8601
1214
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
1215
+ stub_elastic_ping
1216
+ stub_elastic
1217
+ driver.run(default_tag: 'test') do
1218
+ driver.feed(time.to_i, sample_record)
1219
+ end
1220
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
1221
+ end
1222
+
1223
+ def test_error_if_tag_not_in_chunk_keys
1224
+ assert_raise(Fluent::ConfigError) {
1225
+ config = %{
1226
+ <buffer foo>
1227
+ </buffer>
1228
+ }
1229
+ driver.configure(config)
1230
+ }
1231
+ end
1232
+
1233
+ def test_can_use_custom_chunk_along_with_tag
1234
+ config = %{
1235
+ <buffer tag, foo>
1236
+ </buffer>
1237
+ }
1238
+ driver.configure(config)
1239
+ end
1240
+
1241
+ def test_doesnt_add_logstash_timestamp_by_default
1242
+ stub_elastic_ping
1243
+ stub_elastic
1244
+ driver.run(default_tag: 'test') do
1245
+ driver.feed(sample_record)
1246
+ end
1247
+ assert_nil(index_cmds[1]['@timestamp'])
1248
+ end
1249
+
1250
+ def test_adds_timestamp_when_logstash
1251
+ driver.configure("logstash_format true\n")
1252
+ stub_elastic_ping
1253
+ stub_elastic
1254
+ ts = DateTime.now
1255
+ time = Fluent::EventTime.from_time(ts.to_time)
1256
+ driver.run(default_tag: 'test') do
1257
+ driver.feed(time, sample_record)
1258
+ end
1259
+ assert(index_cmds[1].has_key? '@timestamp')
1260
+ assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1261
+ end
1262
+
1263
+ def test_adds_timestamp_when_include_timestamp
1264
+ driver.configure("include_timestamp true\n")
1265
+ stub_elastic_ping
1266
+ stub_elastic
1267
+ ts = DateTime.now
1268
+ time = Fluent::EventTime.from_time(ts.to_time)
1269
+ driver.run(default_tag: 'test') do
1270
+ driver.feed(time, sample_record)
1271
+ end
1272
+ assert(index_cmds[1].has_key? '@timestamp')
1273
+ assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
1274
+ end
1275
+
1276
+ def test_uses_custom_timestamp_when_included_in_record
1277
+ driver.configure("logstash_format true\n")
1278
+ stub_elastic_ping
1279
+ stub_elastic
1280
+ ts = DateTime.new(2001,2,3).iso8601
1281
+ driver.run(default_tag: 'test') do
1282
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1283
+ end
1284
+ assert(index_cmds[1].has_key? '@timestamp')
1285
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1286
+ end
1287
+
1288
+ def test_uses_custom_timestamp_when_included_in_record_without_logstash
1289
+ driver.configure("include_timestamp true\n")
1290
+ stub_elastic_ping
1291
+ stub_elastic
1292
+ ts = DateTime.new(2001,2,3).iso8601
1293
+ driver.run(default_tag: 'test') do
1294
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1295
+ end
1296
+ assert(index_cmds[1].has_key? '@timestamp')
1297
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1298
+ end
1299
+
1300
+ def test_uses_custom_time_key
1301
+ driver.configure("logstash_format true
1302
+ time_key vtm\n")
1303
+ stub_elastic_ping
1304
+ stub_elastic
1305
+ ts = DateTime.new(2001,2,3).iso8601(9)
1306
+ driver.run(default_tag: 'test') do
1307
+ driver.feed(sample_record.merge!('vtm' => ts))
1308
+ end
1309
+ assert(index_cmds[1].has_key? '@timestamp')
1310
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1311
+ end
1312
+
1313
+ def test_uses_custom_time_key_with_format
1314
+ driver.configure("logstash_format true
1315
+ time_key_format %Y-%m-%d %H:%M:%S.%N%z
1316
+ time_key vtm\n")
1317
+ stub_elastic_ping
1318
+ stub_elastic
1319
+ ts = "2001-02-03 13:14:01.673+02:00"
1320
+ driver.run(default_tag: 'test') do
1321
+ driver.feed(sample_record.merge!('vtm' => ts))
1322
+ end
1323
+ assert(index_cmds[1].has_key? '@timestamp')
1324
+ assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1325
+ assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1326
+ end
1327
+
1328
+ def test_uses_custom_time_key_with_format_without_logstash
1329
+ driver.configure("include_timestamp true
1330
+ index_name test
1331
+ time_key_format %Y-%m-%d %H:%M:%S.%N%z
1332
+ time_key vtm\n")
1333
+ stub_elastic_ping
1334
+ stub_elastic
1335
+ ts = "2001-02-03 13:14:01.673+02:00"
1336
+ driver.run(default_tag: 'test') do
1337
+ driver.feed(sample_record.merge!('vtm' => ts))
1338
+ end
1339
+ assert(index_cmds[1].has_key? '@timestamp')
1340
+ assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
1341
+ assert_equal("test", index_cmds[0]['index']['_index'])
1342
+ end
1343
+
1344
+ def test_uses_custom_time_key_exclude_timekey
1345
+ driver.configure("logstash_format true
1346
+ time_key vtm
1347
+ time_key_exclude_timestamp true\n")
1348
+ stub_elastic_ping
1349
+ stub_elastic
1350
+ ts = DateTime.new(2001,2,3).iso8601
1351
+ driver.run(default_tag: 'test') do
1352
+ driver.feed(sample_record.merge!('vtm' => ts))
1353
+ end
1354
+ assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be messing')
1355
+ end
1356
+
1357
+ def test_uses_custom_time_key_format
1358
+ driver.configure("logstash_format true
1359
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1360
+ stub_elastic_ping
1361
+ stub_elastic
1362
+ ts = "2001-02-03T13:14:01.673+02:00"
1363
+ driver.run(default_tag: 'test') do
1364
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1365
+ end
1366
+ assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
1367
+ assert(index_cmds[1].has_key? '@timestamp')
1368
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1369
+ end
1370
+
1371
+ def test_uses_custom_time_key_format_without_logstash
1372
+ driver.configure("include_timestamp true
1373
+ index_name test
1374
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1375
+ stub_elastic_ping
1376
+ stub_elastic
1377
+ ts = "2001-02-03T13:14:01.673+02:00"
1378
+ driver.run(default_tag: 'test') do
1379
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1380
+ end
1381
+ assert_equal("test", index_cmds[0]['index']['_index'])
1382
+ assert(index_cmds[1].has_key? '@timestamp')
1383
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1384
+ end
1385
+
1386
+ data(:default => nil,
1387
+ :custom_tag => 'es_plugin.output.time.error')
1388
+ def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
1389
+ tag_config = tag_for_error ? "time_parse_error_tag #{tag_for_error}" : ''
1390
+ tag_for_error = 'Fluent::ElasticsearchOutput::TimeParser.error' if tag_for_error.nil?
1391
+ driver.configure("logstash_format true
1392
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n#{tag_config}\n")
1393
+ stub_elastic_ping
1394
+ stub_elastic
1395
+
1396
+ ts = "2001/02/03 13:14:01,673+02:00"
1397
+ index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
1398
+
1399
+ flexmock(driver.instance.router).should_receive(:emit_error_event)
1400
+ .with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once
1401
+ driver.run(default_tag: 'test') do
1402
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1403
+ end
1404
+
1405
+ assert_equal(index, index_cmds[0]['index']['_index'])
1406
+ assert(index_cmds[1].has_key? '@timestamp')
1407
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1408
+ end
1409
+
1410
+
1411
+ def test_uses_custom_time_key_format_obscure_format
1412
+ driver.configure("logstash_format true
1413
+ time_key_format %a %b %d %H:%M:%S %Z %Y\n")
1414
+ stub_elastic_ping
1415
+ stub_elastic
1416
+ ts = "Thu Nov 29 14:33:20 GMT 2001"
1417
+ driver.run(default_tag: 'test') do
1418
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1419
+ end
1420
+ assert_equal("logstash-2001.11.29", index_cmds[0]['index']['_index'])
1421
+ assert(index_cmds[1].has_key? '@timestamp')
1422
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1423
+ end
1424
+
1425
+ def test_uses_nanosecond_precision_by_default
1426
+ driver.configure("logstash_format true\n")
1427
+ stub_elastic_ping
1428
+ stub_elastic
1429
+ time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1430
+ driver.run(default_tag: 'test') do
1431
+ driver.feed(time, sample_record)
1432
+ end
1433
+ assert(index_cmds[1].has_key? '@timestamp')
1434
+ assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(9))
1435
+ end
1436
+
1437
+ def test_uses_subsecond_precision_when_configured
1438
+ driver.configure("logstash_format true
1439
+ time_precision 3\n")
1440
+ stub_elastic_ping
1441
+ stub_elastic
1442
+ time = Fluent::EventTime.new(Time.now.to_i, 123456789)
1443
+ driver.run(default_tag: 'test') do
1444
+ driver.feed(time, sample_record)
1445
+ end
1446
+ assert(index_cmds[1].has_key? '@timestamp')
1447
+ assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(3))
1448
+ end
1449
+
1450
+ def test_doesnt_add_tag_key_by_default
1451
+ stub_elastic_ping
1452
+ stub_elastic
1453
+ driver.run(default_tag: 'test') do
1454
+ driver.feed(sample_record)
1455
+ end
1456
+ assert_nil(index_cmds[1]['tag'])
1457
+ end
1458
+
1459
+ def test_adds_tag_key_when_configured
1460
+ driver.configure("include_tag_key true\n")
1461
+ stub_elastic_ping
1462
+ stub_elastic
1463
+ driver.run(default_tag: 'mytag') do
1464
+ driver.feed(sample_record)
1465
+ end
1466
+ assert(index_cmds[1].has_key?('tag'))
1467
+ assert_equal(index_cmds[1]['tag'], 'mytag')
1468
+ end
1469
+
1470
+ def test_adds_id_key_when_configured
1471
+ driver.configure("id_key request_id\n")
1472
+ stub_elastic_ping
1473
+ stub_elastic
1474
+ driver.run(default_tag: 'test') do
1475
+ driver.feed(sample_record)
1476
+ end
1477
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1478
+ end
1479
+
1480
+ class NestedIdKeyTest < self
1481
+ def test_adds_nested_id_key_with_dot
1482
+ driver.configure("id_key nested.request_id\n")
1483
+ stub_elastic_ping
1484
+ stub_elastic
1485
+ driver.run(default_tag: 'test') do
1486
+ driver.feed(nested_sample_record)
1487
+ end
1488
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1489
+ end
1490
+
1491
+ def test_adds_nested_id_key_with_dollar_dot
1492
+ driver.configure("id_key $.nested.request_id\n")
1493
+ stub_elastic_ping
1494
+ stub_elastic
1495
+ driver.run(default_tag: 'test') do
1496
+ driver.feed(nested_sample_record)
1497
+ end
1498
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1499
+ end
1500
+
1501
+ def test_adds_nested_id_key_with_bracket
1502
+ driver.configure("id_key $['nested']['request_id']\n")
1503
+ stub_elastic_ping
1504
+ stub_elastic
1505
+ driver.run(default_tag: 'test') do
1506
+ driver.feed(nested_sample_record)
1507
+ end
1508
+ assert_equal(index_cmds[0]['index']['_id'], '42')
1509
+ end
1510
+ end
1511
+
1512
+ def test_doesnt_add_id_key_if_missing_when_configured
1513
+ driver.configure("id_key another_request_id\n")
1514
+ stub_elastic_ping
1515
+ stub_elastic
1516
+ driver.run(default_tag: 'test') do
1517
+ driver.feed(sample_record)
1518
+ end
1519
+ assert(!index_cmds[0]['index'].has_key?('_id'))
1520
+ end
1521
+
1522
+ def test_adds_id_key_when_not_configured
1523
+ stub_elastic_ping
1524
+ stub_elastic
1525
+ driver.run(default_tag: 'test') do
1526
+ driver.feed(sample_record)
1527
+ end
1528
+ assert(!index_cmds[0]['index'].has_key?('_id'))
1529
+ end
1530
+
1531
+ def test_adds_parent_key_when_configured
1532
+ driver.configure("parent_key parent_id\n")
1533
+ stub_elastic_ping
1534
+ stub_elastic
1535
+ driver.run(default_tag: 'test') do
1536
+ driver.feed(sample_record)
1537
+ end
1538
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1539
+ end
1540
+
1541
+ class NestedParentKeyTest < self
1542
+ def test_adds_nested_parent_key_with_dot
1543
+ driver.configure("parent_key nested.parent_id\n")
1544
+ stub_elastic_ping
1545
+ stub_elastic
1546
+ driver.run(default_tag: 'test') do
1547
+ driver.feed(nested_sample_record)
1548
+ end
1549
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1550
+ end
1551
+
1552
+ def test_adds_nested_parent_key_with_dollar_dot
1553
+ driver.configure("parent_key $.nested.parent_id\n")
1554
+ stub_elastic_ping
1555
+ stub_elastic
1556
+ driver.run(default_tag: 'test') do
1557
+ driver.feed(nested_sample_record)
1558
+ end
1559
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1560
+ end
1561
+
1562
+ def test_adds_nested_parent_key_with_bracket
1563
+ driver.configure("parent_key $['nested']['parent_id']\n")
1564
+ stub_elastic_ping
1565
+ stub_elastic
1566
+ driver.run(default_tag: 'test') do
1567
+ driver.feed(nested_sample_record)
1568
+ end
1569
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
1570
+ end
1571
+ end
1572
+
1573
+ def test_doesnt_add_parent_key_if_missing_when_configured
1574
+ driver.configure("parent_key another_parent_id\n")
1575
+ stub_elastic_ping
1576
+ stub_elastic
1577
+ driver.run(default_tag: 'test') do
1578
+ driver.feed(sample_record)
1579
+ end
1580
+ assert(!index_cmds[0]['index'].has_key?('_parent'))
1581
+ end
1582
+
1583
+ def test_adds_parent_key_when_not_configured
1584
+ stub_elastic_ping
1585
+ stub_elastic
1586
+ driver.run(default_tag: 'test') do
1587
+ driver.feed(sample_record)
1588
+ end
1589
+ assert(!index_cmds[0]['index'].has_key?('_parent'))
1590
+ end
1591
+
1592
+ def test_adds_routing_key_when_configured
1593
+ driver.configure("routing_key routing_id\n")
1594
+ stub_elastic_ping
1595
+ stub_elastic
1596
+ driver.run(default_tag: 'test') do
1597
+ driver.feed(sample_record)
1598
+ end
1599
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1600
+ end
1601
+
1602
+ class NestedRoutingKeyTest < self
1603
+ def test_adds_nested_routing_key_with_dot
1604
+ driver.configure("routing_key nested.routing_id\n")
1605
+ stub_elastic_ping
1606
+ stub_elastic
1607
+ driver.run(default_tag: 'test') do
1608
+ driver.feed(nested_sample_record)
1609
+ end
1610
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1611
+ end
1612
+
1613
+ def test_adds_nested_routing_key_with_dollar_dot
1614
+ driver.configure("routing_key $.nested.routing_id\n")
1615
+ stub_elastic_ping
1616
+ stub_elastic
1617
+ driver.run(default_tag: 'test') do
1618
+ driver.feed(nested_sample_record)
1619
+ end
1620
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1621
+ end
1622
+
1623
+ def test_adds_nested_routing_key_with_bracket
1624
+ driver.configure("routing_key $['nested']['routing_id']\n")
1625
+ stub_elastic_ping
1626
+ stub_elastic
1627
+ driver.run(default_tag: 'test') do
1628
+ driver.feed(nested_sample_record)
1629
+ end
1630
+ assert_equal(index_cmds[0]['index']['_routing'], 'routing')
1631
+ end
1632
+ end
1633
+
1634
+ def test_doesnt_add_routing_key_if_missing_when_configured
1635
+ driver.configure("routing_key another_routing_id\n")
1636
+ stub_elastic_ping
1637
+ stub_elastic
1638
+ driver.run(default_tag: 'test') do
1639
+ driver.feed(sample_record)
1640
+ end
1641
+ assert(!index_cmds[0]['index'].has_key?('_routing'))
1642
+ end
1643
+
1644
+ def test_adds_routing_key_when_not_configured
1645
+ stub_elastic_ping
1646
+ stub_elastic
1647
+ driver.run(default_tag: 'test') do
1648
+ driver.feed(sample_record)
1649
+ end
1650
+ assert(!index_cmds[0]['index'].has_key?('_routing'))
1651
+ end
1652
+
1653
+ def test_remove_one_key
1654
+ driver.configure("remove_keys key1\n")
1655
+ stub_elastic_ping
1656
+ stub_elastic
1657
+ driver.run(default_tag: 'test') do
1658
+ driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1659
+ end
1660
+ assert(!index_cmds[1].has_key?('key1'))
1661
+ assert(index_cmds[1].has_key?('key2'))
1662
+ end
1663
+
1664
+ def test_remove_multi_keys
1665
+ driver.configure("remove_keys key1, key2\n")
1666
+ stub_elastic_ping
1667
+ stub_elastic
1668
+ driver.run(default_tag: 'test') do
1669
+ driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
1670
+ end
1671
+ assert(!index_cmds[1].has_key?('key1'))
1672
+ assert(!index_cmds[1].has_key?('key2'))
1673
+ end
1674
+
1675
+ def test_request_error
1676
+ stub_elastic_ping
1677
+ stub_elastic_unavailable
1678
+ assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
1679
+ driver.run(default_tag: 'test') do
1680
+ driver.feed(sample_record)
1681
+ end
1682
+ }
1683
+ end
1684
+
1685
+ def test_connection_failed_retry
1686
+ connection_resets = 0
1687
+
1688
+ stub_elastic_ping(url="http://localhost:9200").with do |req|
1689
+ connection_resets += 1
1690
+ end
1691
+
1692
+ stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1693
+ raise Faraday::ConnectionFailed, "Test message"
1694
+ end
1695
+
1696
+ driver.run(default_tag: 'test') do
1697
+ driver.feed(sample_record)
1698
+ end
1699
+ assert_equal(connection_resets, 3)
1700
+ end
1701
+
1702
+ def test_reconnect_on_error_enabled
1703
+ connection_resets = 0
1704
+
1705
+ stub_elastic_ping(url="http://localhost:9200").with do |req|
1706
+ connection_resets += 1
1707
+ end
1708
+
1709
+ stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1710
+ raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1711
+ end
1712
+
1713
+ driver.configure("reconnect_on_error true\n")
1714
+
1715
+ assert_raise(ZeroDivisionError) {
1716
+ driver.run(default_tag: 'test', shutdown: false) do
1717
+ driver.feed(sample_record)
1718
+ end
1719
+ }
1720
+
1721
+ assert_raise(Timeout::Error) {
1722
+ driver.run(default_tag: 'test', shutdown: false) do
1723
+ driver.feed(sample_record)
1724
+ end
1725
+ }
1726
+ # FIXME: Consider keywords arguments in #run and how to test this later.
1727
+ # Because v0.14 test driver does not have 1 to 1 correspondence between #run and #flush in tests.
1728
+ assert_equal(connection_resets, 1)
1729
+ end
1730
+
1731
+ def test_reconnect_on_error_disabled
1732
+ connection_resets = 0
1733
+
1734
+ stub_elastic_ping(url="http://localhost:9200").with do |req|
1735
+ connection_resets += 1
1736
+ end
1737
+
1738
+ stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1739
+ raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1740
+ end
1741
+
1742
+ driver.configure("reconnect_on_error false\n")
1743
+
1744
+ assert_raise(ZeroDivisionError) {
1745
+ driver.run(default_tag: 'test', shutdown: false) do
1746
+ driver.feed(sample_record)
1747
+ end
1748
+ }
1749
+
1750
+ assert_raise(Timeout::Error) {
1751
+ driver.run(default_tag: 'test', shutdown: false) do
1752
+ driver.feed(sample_record)
1753
+ end
1754
+ }
1755
+ assert_equal(connection_resets, 1)
1756
+ end
1757
+
1758
+ def test_bulk_error_retags_when_configured
1759
+ driver.configure("retry_tag retry\n")
1760
+ stub_elastic_ping
1761
+ stub_request(:post, 'http://localhost:9200/_bulk')
1762
+ .to_return(lambda do |req|
1763
+ { :status => 200,
1764
+ :headers => { 'Content-Type' => 'json' },
1765
+ :body => %({
1766
+ "took" : 1,
1767
+ "errors" : true,
1768
+ "items" : [
1769
+ {
1770
+ "create" : {
1771
+ "_index" : "foo",
1772
+ "_type" : "bar",
1773
+ "_id" : "abc",
1774
+ "status" : 500,
1775
+ "error" : {
1776
+ "type" : "some unrecognized type",
1777
+ "reason":"some error to cause version mismatch"
1778
+ }
1779
+ }
1780
+ }
1781
+ ]
1782
+ })
1783
+ }
1784
+ end)
1785
+
1786
+ driver.run(default_tag: 'test') do
1787
+ driver.feed(1, sample_record)
1788
+ end
1789
+
1790
+ assert_equal [['retry', 1, sample_record]], driver.events
1791
+ end
1792
+
1793
+ def test_bulk_error
1794
+ stub_elastic_ping
1795
+ stub_request(:post, 'http://localhost:9200/_bulk')
1796
+ .to_return(lambda do |req|
1797
+ { :status => 200,
1798
+ :headers => { 'Content-Type' => 'json' },
1799
+ :body => %({
1800
+ "took" : 1,
1801
+ "errors" : true,
1802
+ "items" : [
1803
+ {
1804
+ "create" : {
1805
+ "_index" : "foo",
1806
+ "_type" : "bar",
1807
+ "_id" : "abc",
1808
+ "status" : 500,
1809
+ "error" : {
1810
+ "type" : "some unrecognized type",
1811
+ "reason":"some error to cause version mismatch"
1812
+ }
1813
+ }
1814
+ },
1815
+ {
1816
+ "create" : {
1817
+ "_index" : "foo",
1818
+ "_type" : "bar",
1819
+ "_id" : "abc",
1820
+ "status" : 201
1821
+ }
1822
+ },
1823
+ {
1824
+ "create" : {
1825
+ "_index" : "foo",
1826
+ "_type" : "bar",
1827
+ "_id" : "abc",
1828
+ "status" : 500,
1829
+ "error" : {
1830
+ "type" : "some unrecognized type",
1831
+ "reason":"some error to cause version mismatch"
1832
+ }
1833
+ }
1834
+ },
1835
+ {
1836
+ "create" : {
1837
+ "_index" : "foo",
1838
+ "_type" : "bar",
1839
+ "_id" : "abc",
1840
+ "_id" : "abc",
1841
+ "status" : 409
1842
+ }
1843
+ }
1844
+ ]
1845
+ })
1846
+ }
1847
+ end)
1848
+
1849
+ driver.run(default_tag: 'test') do
1850
+ driver.feed(1, sample_record)
1851
+ driver.feed(2, sample_record)
1852
+ driver.feed(3, sample_record)
1853
+ driver.feed(4, sample_record)
1854
+ end
1855
+
1856
+ expect = [['test', 1, sample_record],
1857
+ ['test', 3, sample_record]]
1858
+ assert_equal expect, driver.events
1859
+ end
1860
+
1861
+ def test_update_should_not_write_if_theres_no_id
1862
+ driver.configure("write_operation update\n")
1863
+ stub_elastic_ping
1864
+ stub_elastic
1865
+ driver.run(default_tag: 'test') do
1866
+ driver.feed(sample_record)
1867
+ end
1868
+ assert_nil(index_cmds)
1869
+ end
1870
+
1871
+ def test_upsert_should_not_write_if_theres_no_id
1872
+ driver.configure("write_operation upsert\n")
1873
+ stub_elastic_ping
1874
+ stub_elastic
1875
+ driver.run(default_tag: 'test') do
1876
+ driver.feed(sample_record)
1877
+ end
1878
+ assert_nil(index_cmds)
1879
+ end
1880
+
1881
+ def test_create_should_not_write_if_theres_no_id
1882
+ driver.configure("write_operation create\n")
1883
+ stub_elastic_ping
1884
+ stub_elastic
1885
+ driver.run(default_tag: 'test') do
1886
+ driver.feed(sample_record)
1887
+ end
1888
+ assert_nil(index_cmds)
1889
+ end
1890
+
1891
+ def test_update_should_write_update_op_and_doc_as_upsert_is_false
1892
+ driver.configure("write_operation update
1893
+ id_key request_id")
1894
+ stub_elastic_ping
1895
+ stub_elastic
1896
+ driver.run(default_tag: 'test') do
1897
+ driver.feed(sample_record)
1898
+ end
1899
+ assert(index_cmds[0].has_key?("update"))
1900
+ assert(!index_cmds[1]["doc_as_upsert"])
1901
+ assert(!index_cmds[1]["upsert"])
1902
+ end
1903
+
1904
+ def test_update_should_remove_keys_from_doc_when_keys_are_skipped
1905
+ driver.configure("write_operation update
1906
+ id_key request_id
1907
+ remove_keys_on_update parent_id")
1908
+ stub_elastic_ping
1909
+ stub_elastic
1910
+ driver.run(default_tag: 'test') do
1911
+ driver.feed(sample_record)
1912
+ end
1913
+ assert(index_cmds[1]["doc"])
1914
+ assert(!index_cmds[1]["doc"]["parent_id"])
1915
+ end
1916
+
1917
+ def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
1918
+ driver.configure("write_operation upsert
1919
+ id_key request_id")
1920
+ stub_elastic_ping
1921
+ stub_elastic
1922
+ driver.run(default_tag: 'test') do
1923
+ driver.feed(sample_record)
1924
+ end
1925
+ assert(index_cmds[0].has_key?("update"))
1926
+ assert(index_cmds[1]["doc_as_upsert"])
1927
+ assert(!index_cmds[1]["upsert"])
1928
+ end
1929
+
1930
+ def test_upsert_should_write_update_op_upsert_and_doc_when_keys_are_skipped
1931
+ driver.configure("write_operation upsert
1932
+ id_key request_id
1933
+ remove_keys_on_update parent_id")
1934
+ stub_elastic_ping
1935
+ stub_elastic
1936
+ driver.run(default_tag: 'test') do
1937
+ driver.feed(sample_record)
1938
+ end
1939
+ assert(index_cmds[0].has_key?("update"))
1940
+ assert(!index_cmds[1]["doc_as_upsert"])
1941
+ assert(index_cmds[1]["upsert"])
1942
+ assert(index_cmds[1]["doc"])
1943
+ end
1944
+
1945
+ def test_upsert_should_remove_keys_from_doc_when_keys_are_skipped
1946
+ driver.configure("write_operation upsert
1947
+ id_key request_id
1948
+ remove_keys_on_update parent_id")
1949
+ stub_elastic_ping
1950
+ stub_elastic
1951
+ driver.run(default_tag: 'test') do
1952
+ driver.feed(sample_record)
1953
+ end
1954
+ assert(index_cmds[1]["upsert"] != index_cmds[1]["doc"])
1955
+ assert(!index_cmds[1]["doc"]["parent_id"])
1956
+ assert(index_cmds[1]["upsert"]["parent_id"])
1957
+ end
1958
+
1959
+ def test_upsert_should_remove_multiple_keys_when_keys_are_skipped
1960
+ driver.configure("write_operation upsert
1961
+ id_key id
1962
+ remove_keys_on_update foo,baz")
1963
+ stub_elastic_ping
1964
+ stub_elastic
1965
+ driver.run(default_tag: 'test') do
1966
+ driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
1967
+ end
1968
+ assert(
1969
+ index_cmds[1]["doc"] == {
1970
+ "id" => 1,
1971
+ "zip" => "zam",
1972
+ }
1973
+ )
1974
+ assert(
1975
+ index_cmds[1]["upsert"] == {
1976
+ "id" => 1,
1977
+ "foo" => "bar",
1978
+ "baz" => "quix",
1979
+ "zip" => "zam",
1980
+ }
1981
+ )
1982
+ end
1983
+
1984
+ def test_upsert_should_remove_keys_from_when_the_keys_are_in_the_record
1985
+ driver.configure("write_operation upsert
1986
+ id_key id
1987
+ remove_keys_on_update_key keys_to_skip")
1988
+ stub_elastic_ping
1989
+ stub_elastic
1990
+ driver.run(default_tag: 'test') do
1991
+ driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
1992
+ end
1993
+ assert(
1994
+ index_cmds[1]["doc"] == {
1995
+ "id" => 1,
1996
+ "foo" => "bar",
1997
+ }
1998
+ )
1999
+ assert(
2000
+ index_cmds[1]["upsert"] == {
2001
+ "id" => 1,
2002
+ "foo" => "bar",
2003
+ "baz" => "quix",
2004
+ }
2005
+ )
2006
+ end
2007
+
2008
+ def test_upsert_should_remove_keys_from_key_on_record_has_higher_presedence_than_config
2009
+ driver.configure("write_operation upsert
2010
+ id_key id
2011
+ remove_keys_on_update foo,bar
2012
+ remove_keys_on_update_key keys_to_skip")
2013
+ stub_elastic_ping
2014
+ stub_elastic
2015
+ driver.run(default_tag: 'test') do
2016
+ driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
2017
+ end
2018
+ assert(
2019
+ index_cmds[1]["doc"] == {
2020
+ "id" => 1,
2021
+ # we only expect baz to be stripped here, if the config was more important
2022
+ # foo would be stripped too.
2023
+ "foo" => "bar",
2024
+ }
2025
+ )
2026
+ assert(
2027
+ index_cmds[1]["upsert"] == {
2028
+ "id" => 1,
2029
+ "foo" => "bar",
2030
+ "baz" => "quix",
2031
+ }
2032
+ )
2033
+ end
2034
+
2035
+ def test_create_should_write_create_op
2036
+ driver.configure("write_operation create
2037
+ id_key request_id")
2038
+ stub_elastic_ping
2039
+ stub_elastic
2040
+ driver.run(default_tag: 'test') do
2041
+ driver.feed(sample_record)
2042
+ end
2043
+ assert(index_cmds[0].has_key?("create"))
2044
+ end
2045
+
2046
+ end