fluent-plugin-elasticsearch 3.7.0 → 3.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3edaea3928f2cc09c7a908d870afdc2cac2a6d15f2e3f36cc7aa2f27962da79f
4
- data.tar.gz: '09c000b05842db4f08a546f09adb3fe725d50fcf213435d3c7fc0dfa94753551'
3
+ metadata.gz: 66d34ebe18eabf21cd55dbe1c0f3d4f469298e3ada1072849068c1d490e62650
4
+ data.tar.gz: f91f6d38cc821642ddd6bc2fda232534b14d070fa9694db328bdef68d982d93a
5
5
  SHA512:
6
- metadata.gz: 148893479963ccf22397b41be92ea10dbe49e852d54412daa10659e8eb52c8bd1f0ac9546549ec28dde9df53a40f41e6fc45746eb95c10ab8e885e8e38d60537
7
- data.tar.gz: f7e31c37963276c11162d78aa45b52c71feaaf409dd548b72cfd32c652b784008ccc0d0777da0f050817f3d1803023e523c3d693d193a21725aeeb3849518fd6
6
+ metadata.gz: d83715d884fd22a0cb487e0eea16d37679800ce477bfcc888546d1856e48dd8bfa45106826731008b5b3d67cb2408a4d7eca9740a92076ec877a0f999252a512
7
+ data.tar.gz: d5e476b7ec45883dceeecffeb529d43c1b5a16a9a6c203139ed42891a8e702bc74861edcced7bfd87340e4f3150de0f5a7f7d9ecff8840d1dab9e1348db11601
data/History.md CHANGED
@@ -1,6 +1,11 @@
1
1
  ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
2
2
 
3
3
  ### [Unreleased]
4
+ ### 3.7.1
5
+ - Make conpatible for Fluentd v1.8 (#677)
6
+ - Handle flatten_hashes in elasticsearch_dynamic (#675)
7
+ - Handle empty index_date_pattern parameter (#674)
8
+
4
9
  ### 3.7.0
5
10
  - Tweak for cosmetic change (#671)
6
11
  - Fix access to Elasticsearch::Transport::VERSION with explicit top level class path (#670)
@@ -284,7 +284,7 @@ Elasticsearch Input plugin and Elasticsearch output plugin can combine to transf
284
284
  host transferred-cluster.local
285
285
  port 9200
286
286
  index_name ${$.@metadata._index}
287
- # type_name ${$.@metadata._type} # This parameter is optional due to Removal of mapping types since ES7.
287
+ type_name ${$.@metadata._type} # This parameter will be deprecated due to Removal of mapping types since ES7.
288
288
  id_key ${$.@metadata._id} # This parameter is needed for prevent duplicated records.
289
289
  <buffer tag, $.@metadata._index, $.@metadata._type, $.@metadata._id>
290
290
  @type memory # should use file buffer for preventing chunk lost
data/README.md CHANGED
@@ -434,10 +434,15 @@ for example: <logstash-default-{now/d}-000001>. Overriding this changes the roll
434
434
  "now/w{xxxx.ww}" would create weekly rollover indexes instead of daily.
435
435
 
436
436
  This setting only takes effect when combined with the [rollover_index](#rollover_index) setting.
437
+
438
+ And rollover\_index is also used in Lifecycle Index Management(ILM) feature.
437
439
  ```
438
440
  index_date_pattern "now/w{xxxx.ww}" # defaults to "now/d"
439
441
  ```
440
442
 
443
+ If empty string(`""`) is specified in `index_date_patter`, index date pattern is not used.
444
+ Elasticsearch plugin just creates <`index_prefix`-`application_name`-000001> rollover index instead of <`index_prefix`-`application_name`-`{index_date_pattern}`-000001>.
445
+
441
446
  If [customize_template](#customize_template) is set, then this parameter will be in effect otherwise ignored.
442
447
 
443
448
  ### deflector_alias
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '3.7.0'
6
+ s.version = '3.7.1'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -130,7 +130,11 @@ module Fluent::ElasticsearchIndexTemplate
130
130
  def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, host)
131
131
  if rollover_index
132
132
  if !client.indices.exists_alias(:name => deflector_alias_name)
133
- index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
133
+ if index_date_pattern.empty?
134
+ index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-000001>'
135
+ else
136
+ index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
137
+ end
134
138
  indexcreation(index_name_temp, host)
135
139
  body = {}
136
140
  body = rollover_alias_payload(deflector_alias_name) if enable_ilm
@@ -206,7 +206,7 @@ EOC
206
206
  end
207
207
 
208
208
  @alias_indexes = []
209
- if !Fluent::Engine.dry_run_mode
209
+ if !dry_run?
210
210
  if @template_name && @template_file
211
211
  if @rollover_index
212
212
  raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
@@ -259,7 +259,7 @@ EOC
259
259
  end
260
260
 
261
261
  @last_seen_major_version =
262
- if @verify_es_version_at_startup && !Fluent::Engine.dry_run_mode
262
+ if @verify_es_version_at_startup && !dry_run?
263
263
  retry_operate(@max_retry_get_es_version) do
264
264
  detect_es_major_version
265
265
  end
@@ -278,7 +278,7 @@ EOC
278
278
  @type_name = nil
279
279
  end
280
280
 
281
- if @validate_client_version && !Fluent::Engine.dry_run_mode
281
+ if @validate_client_version && !dry_run?
282
282
  if @last_seen_major_version != client_library_version.to_i
283
283
  raise Fluent::ConfigError, <<-EOC
284
284
  Detected ES #{@last_seen_major_version} but you use ES client #{client_library_version}.
@@ -343,6 +343,14 @@ EOC
343
343
  end
344
344
  end
345
345
 
346
+ def dry_run?
347
+ if Fluent::Engine.respond_to?(:dry_run_mode)
348
+ Fluent::Engine.dry_run_mode
349
+ elsif Fluent::Engine.respond_to?(:supervisor_mode)
350
+ Fluent::Engine.supervisor_mode
351
+ end
352
+ end
353
+
346
354
  def placeholder?(name, param)
347
355
  begin
348
356
  placeholder_validate!(name, param)
@@ -132,6 +132,10 @@ module Fluent::Plugin
132
132
  chunk.msgpack_each do |time, record|
133
133
  next unless record.is_a? Hash
134
134
 
135
+ if @flatten_hashes
136
+ record = flatten_record(record)
137
+ end
138
+
135
139
  begin
136
140
  # evaluate all configurations here
137
141
  DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
@@ -672,6 +672,77 @@ class ElasticsearchOutput < Test::Unit::TestCase
672
672
  assert_requested(elastic_request)
673
673
  end
674
674
 
675
+ def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
676
+ cwd = File.dirname(__FILE__)
677
+ template_file = File.join(cwd, 'test_template.json')
678
+
679
+ config = %{
680
+ host logs.google.com
681
+ port 777
682
+ scheme https
683
+ path /es/
684
+ user john
685
+ password doe
686
+ template_name logstash
687
+ template_file #{template_file}
688
+ rollover_index true
689
+ index_date_pattern ""
690
+ deflector_alias myapp_deflector
691
+ enable_ilm true
692
+ }
693
+
694
+ # connection start
695
+ stub_request(:head, "https://logs.google.com:777/es//").
696
+ with(basic_auth: ['john', 'doe']).
697
+ to_return(:status => 200, :body => "", :headers => {})
698
+ # check if template exists
699
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
700
+ with(basic_auth: ['john', 'doe']).
701
+ to_return(:status => 404, :body => "", :headers => {})
702
+ # creation
703
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
704
+ with(basic_auth: ['john', 'doe']).
705
+ to_return(:status => 200, :body => "", :headers => {})
706
+ # check if alias exists
707
+ stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
708
+ with(basic_auth: ['john', 'doe']).
709
+ to_return(:status => 404, :body => "", :headers => {})
710
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
711
+ with(basic_auth: ['john', 'doe']).
712
+ to_return(status: 404, body: "", headers: {})
713
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
714
+ with(basic_auth: ['john', 'doe'],
715
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
716
+ to_return(status: 200, body: "", headers: {})
717
+ # put the alias for the index
718
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
719
+ with(basic_auth: ['john', 'doe']).
720
+ to_return(:status => 200, :body => "", :headers => {})
721
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/_alias/myapp_deflector").
722
+ with(basic_auth: ['john', 'doe'],
723
+ :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
724
+ to_return(:status => 200, :body => "", :headers => {})
725
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
726
+ with(basic_auth: ['john', 'doe']).
727
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
728
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
729
+ with(basic_auth: ['john', 'doe']).
730
+ to_return(:status => 404, :body => "", :headers => {})
731
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
732
+ with(basic_auth: ['john', 'doe'],
733
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
734
+ to_return(:status => 200, :body => "", :headers => {})
735
+
736
+ driver(config)
737
+
738
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
739
+ driver.run(default_tag: 'test') do
740
+ driver.feed(sample_record)
741
+ end
742
+
743
+ assert_requested(elastic_request)
744
+ end
745
+
675
746
  def test_template_create_with_rollover_index_and_custom_ilm
676
747
  cwd = File.dirname(__FILE__)
677
748
  template_file = File.join(cwd, 'test_template.json')
@@ -465,6 +465,40 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
465
465
  assert_equal(2000, total)
466
466
  end
467
467
 
468
+ def test_nested_record_with_flattening_on
469
+ driver.configure("flatten_hashes true
470
+ flatten_hashes_separator |")
471
+
472
+ original_hash = {"foo" => {"bar" => "baz"}, "people" => [
473
+ {"age" => "25", "height" => "1ft"},
474
+ {"age" => "30", "height" => "2ft"}
475
+ ]}
476
+
477
+ expected_output = {"foo|bar"=>"baz", "people" => [
478
+ {"age" => "25", "height" => "1ft"},
479
+ {"age" => "30", "height" => "2ft"}
480
+ ]}
481
+
482
+ stub_elastic
483
+ driver.run(default_tag: 'test') do
484
+ driver.feed(original_hash)
485
+ end
486
+ assert_equal expected_output, index_cmds[1]
487
+ end
488
+
489
+ def test_nested_record_with_flattening_off
490
+ # flattening off by default
491
+
492
+ original_hash = {"foo" => {"bar" => "baz"}}
493
+ expected_output = {"foo" => {"bar" => "baz"}}
494
+
495
+ stub_elastic
496
+ driver.run(default_tag: 'test') do
497
+ driver.feed(original_hash)
498
+ end
499
+ assert_equal expected_output, index_cmds[1]
500
+ end
501
+
468
502
  def test_makes_bulk_request
469
503
  stub_elastic
470
504
  driver.run(default_tag: 'test') do
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.7.0
4
+ version: 3.7.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2019-11-18 00:00:00.000000000 Z
13
+ date: 2019-12-04 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd