fluent-plugin-elasticsearch 4.0.10 → 4.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a12dc6596a56b7486d45129b64187021215433d2e7aa5428e96712fb63a345b5
4
- data.tar.gz: 6fe5296f4a0afa23e863399f6d1cc9eeaa8f70c55e5cb979ca8db1bbf7a8988f
3
+ metadata.gz: c90e0b347a5c90ec23231df927384dadcc4b91b813a4e2c11df23539dfe05628
4
+ data.tar.gz: d6e296c8006870a5f85bd0b97294c69d797fc3f40b574ede7645fcd808ae6356
5
5
  SHA512:
6
- metadata.gz: 4f5d200eb1dee56f9f4fc05efc46cdbf440e0e2df7152e527985b124ac91ff74b0d263bee36f69fddec9f5ccc3730032145d51a95cc22fc00817f909fd4c2a7a
7
- data.tar.gz: 11b0453ea6cbcfaf46e71866a96daf4ceba8f3877dd23e7aa124ea214f056b1b7f56bd8082ae273883a7403a2f1752c64beb377a2a3564c1d30757a8a5f101fa
6
+ metadata.gz: 8466c4e3a13ef5c5ca1bda536a0baf7044fdaa0bc3ae1e1c7e6d0e6cf3dc7596d2b68e3f61ed2fb788f8ef2d8392939b8f63fd6aedc53e5818725bc6e0910aa6
7
+ data.tar.gz: b0572e50952ce49e44b280e94764d0d204d482e8e89cc2124e9a3025c1bd4cf7debe5625ad141337e5f81e6b6816286ee81f931f6b782f0a8cffcbfa0ed12dc3
data/History.md CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 4.0.11
6
+ - Add custom and time placeholders combination testcase for ILM (#781)
7
+ - Really support ILM on `logstash_format` enabled environment (#779)
8
+
5
9
  ### 4.0.10
6
10
  - filter_elasticsearch_genid: Use entire record as hash seed (#777)
7
11
  - Suppress type in meta with suppress_type_name parameter (#774)
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.0.10'
6
+ s.version = '4.0.11'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -761,9 +761,9 @@ EOC
761
761
  begin
762
762
  meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
763
763
  info = if @include_index_in_url
764
- RequestInfo.new(host, meta.delete("_index".freeze), meta["_index".freeze])
764
+ RequestInfo.new(host, meta.delete("_index".freeze), meta.delete("_alias".freeze))
765
765
  else
766
- RequestInfo.new(host, nil, meta["_index".freeze])
766
+ RequestInfo.new(host, nil, meta.delete("_alias".freeze))
767
767
  end
768
768
 
769
769
  if split_request?(bulk_message, info)
@@ -809,7 +809,7 @@ EOC
809
809
  end
810
810
 
811
811
  def process_message(tag, meta, header, time, record, extracted_values)
812
- logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline, _ilm_policy_id = extracted_values
812
+ logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, application_name, pipeline, _ilm_policy_id = extracted_values
813
813
 
814
814
  if @flatten_hashes
815
815
  record = flatten_record(record)
@@ -832,17 +832,19 @@ EOC
832
832
 
833
833
  target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
834
834
  if target_index_parent && target_index_parent[target_index_child_key]
835
- target_index = target_index_parent.delete(target_index_child_key)
835
+ target_index_alias = target_index = target_index_parent.delete(target_index_child_key)
836
836
  elsif @logstash_format
837
837
  dt = dt.new_offset(0) if @utc_index
838
838
  target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
839
+ target_index_alias = "#{logstash_prefix}#{@logstash_prefix_separator}#{application_name}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
839
840
  else
840
- target_index = index_name
841
+ target_index_alias = target_index = index_name
841
842
  end
842
843
 
843
844
  # Change target_index to lower-case since Elasticsearch doesn't
844
845
  # allow upper-case characters in index names.
845
846
  target_index = target_index.downcase
847
+ target_index_alias = target_index_alias.downcase
846
848
  if @include_tag_key
847
849
  record[@tag_key] = tag
848
850
  end
@@ -877,6 +879,7 @@ EOC
877
879
  meta.clear
878
880
  meta["_index".freeze] = target_index
879
881
  meta["_type".freeze] = target_type unless @last_seen_major_version >= 8
882
+ meta["_alias".freeze] = target_index_alias
880
883
 
881
884
  if @pipeline
882
885
  meta["pipeline".freeze] = pipeline
@@ -1053,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1053
1053
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1054
1054
  end
1055
1055
 
1056
+ def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
1057
+ cwd = File.dirname(__FILE__)
1058
+ template_file = File.join(cwd, 'test_template.json')
1059
+
1060
+ config = %{
1061
+ host logs.google.com
1062
+ port 777
1063
+ scheme https
1064
+ path /es/
1065
+ user john
1066
+ password doe
1067
+ template_name logstash
1068
+ template_file #{template_file}
1069
+ index_date_pattern now/w{xxxx.ww}
1070
+ enable_ilm true
1071
+ logstash_format true
1072
+ application_name log
1073
+ }
1074
+
1075
+ date_str = Time.now.strftime("%Y.%m.%d")
1076
+ # connection start
1077
+ stub_request(:head, "https://logs.google.com:777/es//").
1078
+ with(basic_auth: ['john', 'doe']).
1079
+ to_return(:status => 200, :body => "", :headers => {})
1080
+ # check if template exists
1081
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1082
+ with(basic_auth: ['john', 'doe']).
1083
+ to_return(:status => 404, :body => "", :headers => {})
1084
+ # creation
1085
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1086
+ with(basic_auth: ['john', 'doe']).
1087
+ to_return(:status => 200, :body => "", :headers => {})
1088
+ # check if alias exists
1089
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-log-#{date_str}").
1090
+ with(basic_auth: ['john', 'doe']).
1091
+ to_return(:status => 404, :body => "", :headers => {})
1092
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1093
+ with(basic_auth: ['john', 'doe']).
1094
+ to_return(status: 404, body: "", headers: {})
1095
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1096
+ with(basic_auth: ['john', 'doe'],
1097
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
1098
+ to_return(status: 200, body: "", headers: {})
1099
+ # put the alias for the index
1100
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1101
+ with(basic_auth: ['john', 'doe']).
1102
+ to_return(:status => 200, :body => "", :headers => {})
1103
+
1104
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-log-#{date_str}").
1105
+ with(basic_auth: ['john', 'doe'],
1106
+ body: "{\"aliases\":{\"logstash-log-#{date_str}\":{\"is_write_index\":true}}}").
1107
+ to_return(status: 200, body: "", headers: {})
1108
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1109
+ with(basic_auth: ['john', 'doe']).
1110
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1111
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1112
+ with(basic_auth: ['john', 'doe']).
1113
+ to_return(:status => 404, :body => "", :headers => {})
1114
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1115
+ with(basic_auth: ['john', 'doe'],
1116
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1117
+ to_return(:status => 200, :body => "", :headers => {})
1118
+
1119
+ driver(config)
1120
+
1121
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1122
+ driver.run(default_tag: 'test') do
1123
+ driver.feed(sample_record)
1124
+ end
1125
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}", times: 1)
1126
+
1127
+ assert_requested(elastic_request)
1128
+ end
1129
+
1056
1130
  def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
1057
1131
  cwd = File.dirname(__FILE__)
1058
1132
  template_file = File.join(cwd, 'test_template.json')
@@ -1607,6 +1681,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1607
1681
 
1608
1682
  assert_requested(elastic_request)
1609
1683
  end
1684
+
1685
+ def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
1686
+ cwd = File.dirname(__FILE__)
1687
+ template_file = File.join(cwd, 'test_template.json')
1688
+
1689
+ config = Fluent::Config::Element.new(
1690
+ 'ROOT', '', {
1691
+ '@type' => 'elasticsearch',
1692
+ 'host' => 'logs.google.com',
1693
+ 'port' => 777,
1694
+ 'scheme' => "https",
1695
+ 'path' => "/es/",
1696
+ 'user' => 'john',
1697
+ 'password' => 'doe',
1698
+ 'template_name' => 'logstash',
1699
+ 'template_file' => "#{template_file}",
1700
+ 'index_date_pattern' => 'now/w{xxxx.ww}',
1701
+ 'index_name' => "${taskDef}-%Y.%m",
1702
+ 'enable_ilm' => true,
1703
+ }, [
1704
+ Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
1705
+ 'chunk_keys' => ['tag', 'time', 'taskDef'],
1706
+ 'timekey' => 3600,
1707
+ }, [])
1708
+ ]
1709
+ )
1710
+
1711
+ task_def_value = "task_definition"
1712
+ date_str = Time.now.strftime("%Y.%m")
1713
+ # connection start
1714
+ stub_request(:head, "https://logs.google.com:777/es//").
1715
+ with(basic_auth: ['john', 'doe']).
1716
+ to_return(:status => 200, :body => "", :headers => {})
1717
+ # check if template exists
1718
+ stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1719
+ with(basic_auth: ['john', 'doe']).
1720
+ to_return(:status => 404, :body => "", :headers => {})
1721
+ # creation
1722
+ stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1723
+ with(basic_auth: ['john', 'doe'],
1724
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
1725
+ to_return(:status => 200, :body => "", :headers => {})
1726
+ # check if alias exists
1727
+ stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
1728
+ with(basic_auth: ['john', 'doe']).
1729
+ to_return(:status => 404, :body => "", :headers => {})
1730
+ # put the alias for the index
1731
+ stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1732
+ with(basic_auth: ['john', 'doe']).
1733
+ to_return(:status => 200, :body => "", :headers => {})
1734
+ stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
1735
+ with(basic_auth: ['john', 'doe'],
1736
+ :body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
1737
+ to_return(:status => 200, :body => "", :headers => {})
1738
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1739
+ with(basic_auth: ['john', 'doe']).
1740
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1741
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1742
+ with(basic_auth: ['john', 'doe']).
1743
+ to_return(:status => 404, :body => "", :headers => {})
1744
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1745
+ with(basic_auth: ['john', 'doe'],
1746
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1747
+ to_return(:status => 200, :body => "", :headers => {})
1748
+
1749
+ driver(config)
1750
+
1751
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1752
+ driver.run(default_tag: 'test') do
1753
+ driver.feed(sample_record.merge("taskDef" => task_def_value))
1754
+ end
1755
+ assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
1756
+
1757
+ assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
1758
+
1759
+ assert_requested(elastic_request)
1760
+ end
1610
1761
  end
1611
1762
 
1612
1763
  def test_custom_template_create
@@ -1863,11 +2014,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1863
2014
  to_return(:status => 200, :body => "", :headers => {})
1864
2015
  # check if alias exists
1865
2016
  timestr = Time.now.strftime("%Y.%m.%d")
1866
- stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
2017
+ stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
1867
2018
  with(basic_auth: ['john', 'doe']).
1868
2019
  to_return(:status => 404, :body => "", :headers => {})
1869
2020
  # put the alias for the index
1870
- stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
2021
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-myapp-#{timestr}").
1871
2022
  with(basic_auth: ['john', 'doe']).
1872
2023
  to_return(:status => 200, :body => "", :headers => {})
1873
2024
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.10
4
+ version: 4.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-07-06 00:00:00.000000000 Z
13
+ date: 2020-07-09 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd