fluent-plugin-elasticsearch 4.2.1 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +6 -0
- data/README.md +13 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_index_template.rb +12 -9
- data/lib/fluent/plugin/out_elasticsearch.rb +10 -8
- data/test/plugin/test_out_elasticsearch.rb +207 -28
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d54b3f207af54f6fb5d3119f2d6d649478e25471fda03c56e00f9206ae72914e
|
|
4
|
+
data.tar.gz: 82eea8ed3938919f6860582698ce1253214531c806c132723e8845adfb45e2e0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c4fd2785f716d97cb8c44845c51f933f7d0ee69fe3e2911f85bf621d043015e815639350e363ce42866b224905fc20ae12b3666d302e5f1f07937f047123b20d
|
|
7
|
+
data.tar.gz: 15d34dc457c5a98b77f2142b271d3f4ef5f611e38caeb4f225f12dcfc2c3771dea0153bae3ebe99a83cc71c6e414d67e39fea1d2fa1fa430049f90839c677578
|
data/History.md
CHANGED
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
### [Unreleased]
|
|
4
4
|
|
|
5
|
+
### 4.2.2
|
|
6
|
+
- Remove unnecessary nil check (#826)
|
|
7
|
+
- Fix ILM rollover index template pattern to apply index_separator (#825)
|
|
8
|
+
- Fix ILM rollover alias creation when a placeholder is used in index_name (#823)
|
|
9
|
+
- Add a note about the pitfalls of per-date indexes used with ILM (#822)
|
|
10
|
+
|
|
5
11
|
### 4.2.1
|
|
6
12
|
- Update a broken link (#821)
|
|
7
13
|
- Include chunk_id in records (#820)
|
data/README.md
CHANGED
|
@@ -1881,6 +1881,16 @@ template_name your-fluentd-template
|
|
|
1881
1881
|
template_file /path/to/fluentd-template.json
|
|
1882
1882
|
```
|
|
1883
1883
|
|
|
1884
|
+
Note that if you create a new set of indexes every day, the elasticsearch ILM policy system will treat each day separately and will always
|
|
1885
|
+
maintain a separate active write index for each day.
|
|
1886
|
+
|
|
1887
|
+
If you have a rollover based on max_age, it will continue to roll the indexes for prior dates even if no new documents are indexed. If you want
|
|
1888
|
+
to delete indexes after a period of time, the ILM policy will never delete the current write index regardless of its age, so you would need a separate
|
|
1889
|
+
system, such as curator, to actually delete the old indexes.
|
|
1890
|
+
|
|
1891
|
+
For this reason, if you put the date into the index names with ILM you should only rollover based on size or number of documents and may need to use
|
|
1892
|
+
curator to actually delete old indexes.
|
|
1893
|
+
|
|
1884
1894
|
#### Fixed ILM indices
|
|
1885
1895
|
|
|
1886
1896
|
Also, users can use fixed ILM indices configuration.
|
|
@@ -1996,4 +2006,7 @@ Install dev dependencies:
|
|
|
1996
2006
|
$ gem install bundler
|
|
1997
2007
|
$ bundle install
|
|
1998
2008
|
$ bundle exec rake test
|
|
2009
|
+
# To just run the test you are working on:
|
|
2010
|
+
$ bundle exec rake test TEST=test/plugin/test_out_elasticsearch.rb TESTOPTS='--verbose --name=test_custom_template_with_rollover_index_create_and_custom_ilm'
|
|
2011
|
+
|
|
1999
2012
|
```
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |s|
|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
|
6
|
-
s.version = '4.2.
|
|
6
|
+
s.version = '4.2.2'
|
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
|
@@ -73,14 +73,15 @@ module Fluent::ElasticsearchIndexTemplate
|
|
|
73
73
|
end
|
|
74
74
|
end
|
|
75
75
|
|
|
76
|
-
def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
|
|
76
|
+
def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
|
|
77
77
|
inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
|
|
78
78
|
if overwrite
|
|
79
79
|
template_put(inject_template_name,
|
|
80
80
|
enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
|
|
81
81
|
target_index,
|
|
82
82
|
ilm_policy_id,
|
|
83
|
-
get_template(template_file)
|
|
83
|
+
get_template(template_file),
|
|
84
|
+
index_separator) :
|
|
84
85
|
get_template(template_file), host)
|
|
85
86
|
|
|
86
87
|
log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
|
|
@@ -91,7 +92,8 @@ module Fluent::ElasticsearchIndexTemplate
|
|
|
91
92
|
enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
|
|
92
93
|
target_index,
|
|
93
94
|
ilm_policy_id,
|
|
94
|
-
get_template(template_file)
|
|
95
|
+
get_template(template_file),
|
|
96
|
+
index_separator) :
|
|
95
97
|
get_template(template_file), host)
|
|
96
98
|
log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
|
|
97
99
|
else
|
|
@@ -99,14 +101,15 @@ module Fluent::ElasticsearchIndexTemplate
|
|
|
99
101
|
end
|
|
100
102
|
end
|
|
101
103
|
|
|
102
|
-
def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
|
|
104
|
+
def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
|
|
103
105
|
template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
|
|
104
106
|
custom_template = if enable_ilm
|
|
105
107
|
inject_ilm_settings_to_template(deflector_alias_name,
|
|
106
108
|
target_index,
|
|
107
109
|
ilm_policy_id,
|
|
108
110
|
get_custom_template(template_file,
|
|
109
|
-
customize_template)
|
|
111
|
+
customize_template),
|
|
112
|
+
index_separator)
|
|
110
113
|
else
|
|
111
114
|
get_custom_template(template_file, customize_template)
|
|
112
115
|
end
|
|
@@ -127,9 +130,9 @@ module Fluent::ElasticsearchIndexTemplate
|
|
|
127
130
|
enable_ilm ? deflector_alias_name : template_name
|
|
128
131
|
end
|
|
129
132
|
|
|
130
|
-
def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
|
|
133
|
+
def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
|
|
131
134
|
log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
|
|
132
|
-
template['index_patterns'] = "#{target_index}
|
|
135
|
+
template['index_patterns'] = "#{target_index}#{index_separator}*"
|
|
133
136
|
if @use_legacy_template
|
|
134
137
|
template.delete('template') if template.include?('template')
|
|
135
138
|
# Prepare settings Hash
|
|
@@ -140,7 +143,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
|
140
143
|
log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
|
|
141
144
|
end
|
|
142
145
|
template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
|
143
|
-
template['order'] = template['order'] ? template['order'] + target_index.
|
|
146
|
+
template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
|
|
144
147
|
else
|
|
145
148
|
# Prepare template.settings Hash
|
|
146
149
|
if !template['template'].key?('settings')
|
|
@@ -150,7 +153,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
|
150
153
|
log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
|
|
151
154
|
end
|
|
152
155
|
template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
|
153
|
-
template['priority'] = template['priority'] ? template['priority'] + target_index.
|
|
156
|
+
template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
|
|
154
157
|
end
|
|
155
158
|
template
|
|
156
159
|
end
|
|
@@ -259,7 +259,7 @@ EOC
|
|
|
259
259
|
template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
|
|
260
260
|
end
|
|
261
261
|
verify_ilm_working if @enable_ilm
|
|
262
|
-
end
|
|
262
|
+
end
|
|
263
263
|
if @templates
|
|
264
264
|
retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
|
|
265
265
|
templates_hash_install(@templates, @template_overwrite)
|
|
@@ -982,22 +982,24 @@ EOC
|
|
|
982
982
|
|
|
983
983
|
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
|
|
984
984
|
if template_name && @template_file
|
|
985
|
-
if !@logstash_format && @alias_indexes.include?
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
985
|
+
if !@logstash_format && (deflector_alias.nil? || (@alias_indexes.include? deflector_alias)) && (@template_names.include? template_name)
|
|
986
|
+
if deflector_alias
|
|
987
|
+
log.debug("Index alias #{deflector_alias} and template #{template_name} already exist (cached)")
|
|
988
|
+
else
|
|
989
|
+
log.debug("Template #{template_name} already exists (cached)")
|
|
990
|
+
end
|
|
989
991
|
else
|
|
990
992
|
retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
|
|
991
993
|
if customize_template
|
|
992
|
-
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
|
|
994
|
+
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
|
|
993
995
|
else
|
|
994
|
-
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
|
|
996
|
+
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
|
|
995
997
|
end
|
|
996
998
|
ilm_policy = @ilm_policies[ilm_policy_id] || {}
|
|
997
999
|
create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
|
|
998
1000
|
end
|
|
999
1001
|
@alias_indexes << deflector_alias unless deflector_alias.nil?
|
|
1000
|
-
@template_names << template_name
|
|
1002
|
+
@template_names << template_name
|
|
1001
1003
|
end
|
|
1002
1004
|
end
|
|
1003
1005
|
end
|
|
@@ -1775,30 +1775,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1775
1775
|
with(basic_auth: ['john', 'doe']).
|
|
1776
1776
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1777
1777
|
# check if alias exists
|
|
1778
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-
|
|
1778
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-tag1").
|
|
1779
1779
|
with(basic_auth: ['john', 'doe']).
|
|
1780
1780
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1781
|
-
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-
|
|
1781
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
|
|
1782
1782
|
with(basic_auth: ['john', 'doe']).
|
|
1783
1783
|
to_return(status: 404, body: "", headers: {})
|
|
1784
1784
|
if use_legacy_template_flag
|
|
1785
|
-
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-
|
|
1785
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
|
|
1786
1786
|
with(basic_auth: ['john', 'doe'],
|
|
1787
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-
|
|
1787
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-tag1-*\",\"order\":52}").
|
|
1788
1788
|
to_return(status: 200, body: "", headers: {})
|
|
1789
1789
|
else
|
|
1790
|
-
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-
|
|
1790
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag1").
|
|
1791
1791
|
with(basic_auth: ['john', 'doe'],
|
|
1792
|
-
body: "{\"index_patterns\":\"logstash-
|
|
1792
|
+
body: "{\"index_patterns\":\"logstash-tag1-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
|
1793
1793
|
to_return(status: 200, body: "", headers: {})
|
|
1794
1794
|
end
|
|
1795
1795
|
# put the alias for the index
|
|
1796
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-
|
|
1796
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag1-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1797
1797
|
with(basic_auth: ['john', 'doe']).
|
|
1798
1798
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1799
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-
|
|
1799
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag1-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-tag1").
|
|
1800
1800
|
with(basic_auth: ['john', 'doe'],
|
|
1801
|
-
:body => "{\"aliases\":{\"logstash-
|
|
1801
|
+
:body => "{\"aliases\":{\"logstash-tag1\":{\"is_write_index\":true}}}").
|
|
1802
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1803
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1804
|
+
with(basic_auth: ['john', 'doe']).
|
|
1805
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1806
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1807
|
+
with(basic_auth: ['john', 'doe']).
|
|
1808
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1809
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1810
|
+
with(basic_auth: ['john', 'doe'],
|
|
1811
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1812
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1813
|
+
# check if alias exists
|
|
1814
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-tag2").
|
|
1815
|
+
with(basic_auth: ['john', 'doe']).
|
|
1816
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1817
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
|
|
1818
|
+
with(basic_auth: ['john', 'doe']).
|
|
1819
|
+
to_return(status: 404, body: "", headers: {})
|
|
1820
|
+
if use_legacy_template_flag
|
|
1821
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
|
|
1822
|
+
with(basic_auth: ['john', 'doe'],
|
|
1823
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag2\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-tag2-*\",\"order\":52}").
|
|
1824
|
+
to_return(status: 200, body: "", headers: {})
|
|
1825
|
+
else
|
|
1826
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-tag2").
|
|
1827
|
+
with(basic_auth: ['john', 'doe'],
|
|
1828
|
+
body: "{\"index_patterns\":\"logstash-tag2-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-tag2\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
|
1829
|
+
to_return(status: 200, body: "", headers: {})
|
|
1830
|
+
end # put the alias for the index
|
|
1831
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag2-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1832
|
+
with(basic_auth: ['john', 'doe']).
|
|
1833
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1834
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-tag2-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-tag2").
|
|
1835
|
+
with(basic_auth: ['john', 'doe'],
|
|
1836
|
+
:body => "{\"aliases\":{\"logstash-tag2\":{\"is_write_index\":true}}}").
|
|
1802
1837
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1803
1838
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1804
1839
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -1815,11 +1850,104 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1815
1850
|
|
|
1816
1851
|
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1817
1852
|
driver.run(default_tag: 'test') do
|
|
1818
|
-
driver.feed(sample_record)
|
|
1853
|
+
driver.feed('tag1', event_time, sample_record)
|
|
1854
|
+
driver.feed('tag2', event_time, sample_record)
|
|
1855
|
+
driver.feed('tag1', event_time, sample_record)
|
|
1856
|
+
driver.feed('tag2', event_time, sample_record)
|
|
1857
|
+
end
|
|
1858
|
+
assert_equal('logstash-tag2', index_cmds.first['index']['_index'])
|
|
1859
|
+
|
|
1860
|
+
assert_equal ["logstash-tag1", "logstash-tag2"], driver.instance.alias_indexes
|
|
1861
|
+
|
|
1862
|
+
assert_requested(elastic_request)
|
|
1863
|
+
end
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
data("legacy_template" => [true, "_template"],
|
|
1867
|
+
"new_template" => [false, "_index_template"])
|
|
1868
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders_and_index_separator(data)
|
|
1869
|
+
use_legacy_template_flag, endpoint = data
|
|
1870
|
+
cwd = File.dirname(__FILE__)
|
|
1871
|
+
template_file = if use_legacy_template_flag
|
|
1872
|
+
File.join(cwd, 'test_template.json')
|
|
1873
|
+
else
|
|
1874
|
+
File.join(cwd, 'test_index_template.json')
|
|
1875
|
+
end
|
|
1876
|
+
|
|
1877
|
+
config = %{
|
|
1878
|
+
host logs.google.com
|
|
1879
|
+
port 777
|
|
1880
|
+
scheme https
|
|
1881
|
+
path /es/
|
|
1882
|
+
user john
|
|
1883
|
+
password doe
|
|
1884
|
+
template_name logstash
|
|
1885
|
+
template_file #{template_file}
|
|
1886
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1887
|
+
index_name logstash.${tag}
|
|
1888
|
+
enable_ilm true
|
|
1889
|
+
index_separator "."
|
|
1890
|
+
use_legacy_template #{use_legacy_template_flag}
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
# connection start
|
|
1894
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1895
|
+
with(basic_auth: ['john', 'doe']).
|
|
1896
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1897
|
+
# check if template exists
|
|
1898
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
|
1899
|
+
with(basic_auth: ['john', 'doe']).
|
|
1900
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1901
|
+
# creation
|
|
1902
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
|
1903
|
+
with(basic_auth: ['john', 'doe']).
|
|
1904
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1905
|
+
# check if alias exists
|
|
1906
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash.tag1").
|
|
1907
|
+
with(basic_auth: ['john', 'doe']).
|
|
1908
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1909
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
|
|
1910
|
+
with(basic_auth: ['john', 'doe']).
|
|
1911
|
+
to_return(status: 404, body: "", headers: {})
|
|
1912
|
+
if use_legacy_template_flag
|
|
1913
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
|
|
1914
|
+
with(basic_auth: ['john', 'doe'],
|
|
1915
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash.tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash.tag1.*\",\"order\":52}").
|
|
1916
|
+
to_return(status: 200, body: "", headers: {})
|
|
1917
|
+
else
|
|
1918
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash.tag1").
|
|
1919
|
+
with(basic_auth: ['john', 'doe'],
|
|
1920
|
+
body: "{\"index_patterns\":\"logstash.tag1.*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash.tag1\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
|
1921
|
+
to_return(status: 200, body: "", headers: {})
|
|
1819
1922
|
end
|
|
1820
|
-
|
|
1923
|
+
# put the alias for the index
|
|
1924
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash.tag1.default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1925
|
+
with(basic_auth: ['john', 'doe']).
|
|
1926
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1927
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash.tag1.default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash.tag1").
|
|
1928
|
+
with(basic_auth: ['john', 'doe'],
|
|
1929
|
+
:body => "{\"aliases\":{\"logstash.tag1\":{\"is_write_index\":true}}}").
|
|
1930
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1931
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1932
|
+
with(basic_auth: ['john', 'doe']).
|
|
1933
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1934
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1935
|
+
with(basic_auth: ['john', 'doe']).
|
|
1936
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1937
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1938
|
+
with(basic_auth: ['john', 'doe'],
|
|
1939
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1940
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1821
1941
|
|
|
1822
|
-
|
|
1942
|
+
driver(config)
|
|
1943
|
+
|
|
1944
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1945
|
+
driver.run(default_tag: 'test') do
|
|
1946
|
+
driver.feed('tag1', event_time, sample_record)
|
|
1947
|
+
end
|
|
1948
|
+
assert_equal('logstash.tag1', index_cmds.first['index']['_index'])
|
|
1949
|
+
|
|
1950
|
+
assert_equal ["logstash.tag1"], driver.instance.alias_indexes
|
|
1823
1951
|
|
|
1824
1952
|
assert_requested(elastic_request)
|
|
1825
1953
|
end
|
|
@@ -2202,7 +2330,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2202
2330
|
use_legacy_template #{use_legacy_template_flag}
|
|
2203
2331
|
}
|
|
2204
2332
|
|
|
2205
|
-
timestr = Time.now.strftime("%Y.%m.%d")
|
|
2333
|
+
timestr = Time.now.getutc.strftime("%Y.%m.%d")
|
|
2206
2334
|
# connection start
|
|
2207
2335
|
stub_request(:head, "https://logs.google.com:777/es//").
|
|
2208
2336
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -2478,6 +2606,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2478
2606
|
stub_request(:head, "https://logs.google.com:777/es//").
|
|
2479
2607
|
with(basic_auth: ['john', 'doe']).
|
|
2480
2608
|
to_return(:status => 200, :body => "", :headers => {})
|
|
2609
|
+
|
|
2610
|
+
# test-tag1
|
|
2481
2611
|
# check if template exists
|
|
2482
2612
|
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
|
2483
2613
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -2487,31 +2617,78 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2487
2617
|
with(basic_auth: ['john', 'doe']).
|
|
2488
2618
|
to_return(:status => 200, :body => "", :headers => {})
|
|
2489
2619
|
# creation of index which can rollover
|
|
2490
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-
|
|
2620
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag1-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
2621
|
+
with(basic_auth: ['john', 'doe']).
|
|
2622
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
2623
|
+
# check if alias exists
|
|
2624
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-test-tag1").
|
|
2625
|
+
with(basic_auth: ['john', 'doe']).
|
|
2626
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
2627
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
|
|
2628
|
+
with(basic_auth: ['john', 'doe']).
|
|
2629
|
+
to_return(status: 404, body: "", headers: {})
|
|
2630
|
+
if use_legacy_template_flag
|
|
2631
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
|
|
2632
|
+
with(basic_auth: ['john', 'doe'],
|
|
2633
|
+
body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag1\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-test-tag1-*\"}").
|
|
2634
|
+
to_return(status: 200, body: "", headers: {})
|
|
2635
|
+
else
|
|
2636
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag1").
|
|
2637
|
+
with(basic_auth: ['john', 'doe'],
|
|
2638
|
+
body: "{\"priority\":108,\"index_patterns\":\"mylogs-test-tag1-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag1\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
|
2639
|
+
to_return(status: 200, body: "", headers: {})
|
|
2640
|
+
end
|
|
2641
|
+
# put the alias for the index
|
|
2642
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag1-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-test-tag1").
|
|
2643
|
+
with(basic_auth: ['john', 'doe'],
|
|
2644
|
+
:body => "{\"aliases\":{\"mylogs-test-tag1\":{\"is_write_index\":true}}}").
|
|
2645
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
2646
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
2647
|
+
with(basic_auth: ['john', 'doe']).
|
|
2648
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
2649
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
2650
|
+
with(basic_auth: ['john', 'doe']).
|
|
2651
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
2652
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
2653
|
+
with(basic_auth: ['john', 'doe'],
|
|
2654
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
2655
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
2656
|
+
|
|
2657
|
+
# test-tag2
|
|
2658
|
+
# check if template exists
|
|
2659
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
2660
|
+
with(basic_auth: ['john', 'doe']).
|
|
2661
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
2662
|
+
# creation
|
|
2663
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
2664
|
+
with(basic_auth: ['john', 'doe']).
|
|
2665
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
2666
|
+
# creation of index which can rollover
|
|
2667
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag2-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
2491
2668
|
with(basic_auth: ['john', 'doe']).
|
|
2492
2669
|
to_return(:status => 200, :body => "", :headers => {})
|
|
2493
2670
|
# check if alias exists
|
|
2494
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-
|
|
2671
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-test-tag2").
|
|
2495
2672
|
with(basic_auth: ['john', 'doe']).
|
|
2496
2673
|
to_return(:status => 404, :body => "", :headers => {})
|
|
2497
|
-
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-
|
|
2674
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
|
|
2498
2675
|
with(basic_auth: ['john', 'doe']).
|
|
2499
2676
|
to_return(status: 404, body: "", headers: {})
|
|
2500
2677
|
if use_legacy_template_flag
|
|
2501
|
-
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-
|
|
2678
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
|
|
2502
2679
|
with(basic_auth: ['john', 'doe'],
|
|
2503
|
-
body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-
|
|
2680
|
+
body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag2\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-test-tag2-*\"}").
|
|
2504
2681
|
to_return(status: 200, body: "", headers: {})
|
|
2505
2682
|
else
|
|
2506
|
-
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-
|
|
2683
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-test-tag2").
|
|
2507
2684
|
with(basic_auth: ['john', 'doe'],
|
|
2508
|
-
body: "{\"priority\":108,\"index_patterns\":\"mylogs-
|
|
2685
|
+
body: "{\"priority\":108,\"index_patterns\":\"mylogs-test-tag2-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-test-tag2\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
|
2509
2686
|
to_return(status: 200, body: "", headers: {})
|
|
2510
2687
|
end
|
|
2511
2688
|
# put the alias for the index
|
|
2512
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-
|
|
2689
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-test-tag2-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-test-tag2").
|
|
2513
2690
|
with(basic_auth: ['john', 'doe'],
|
|
2514
|
-
:body => "{\"aliases\":{\"mylogs-
|
|
2691
|
+
:body => "{\"aliases\":{\"mylogs-test-tag2\":{\"is_write_index\":true}}}").
|
|
2515
2692
|
to_return(:status => 200, :body => "", :headers => {})
|
|
2516
2693
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
2517
2694
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -2527,12 +2704,14 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2527
2704
|
driver(config)
|
|
2528
2705
|
|
|
2529
2706
|
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
2530
|
-
driver.run(default_tag: '
|
|
2531
|
-
driver.feed(sample_record)
|
|
2707
|
+
driver.run(default_tag: 'test') do
|
|
2708
|
+
driver.feed("test-tag1", event_time, sample_record)
|
|
2709
|
+
driver.feed("test-tag2", event_time, sample_record)
|
|
2710
|
+
driver.feed("test-tag1", event_time, sample_record)
|
|
2711
|
+
driver.feed("test-tag2", event_time, sample_record)
|
|
2532
2712
|
end
|
|
2533
|
-
assert_equal('mylogs-
|
|
2534
|
-
|
|
2535
|
-
assert_equal ["mylogs-custom-test"], driver.instance.alias_indexes
|
|
2713
|
+
assert_equal('mylogs-test-tag2', index_cmds.first['index']['_index'])
|
|
2714
|
+
assert_equal ["mylogs-test-tag1", "mylogs-test-tag2"], driver.instance.alias_indexes
|
|
2536
2715
|
|
|
2537
2716
|
assert_requested(elastic_request)
|
|
2538
2717
|
end
|
|
@@ -4344,7 +4523,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
4344
4523
|
stub_elastic
|
|
4345
4524
|
|
|
4346
4525
|
ts = "2001/02/03 13:14:01,673+02:00"
|
|
4347
|
-
index = "logstash-#{
|
|
4526
|
+
index = "logstash-#{Time.now.getutc.strftime("%Y.%m.%d")}"
|
|
4348
4527
|
|
|
4349
4528
|
flexmock(driver.instance.router).should_receive(:emit_error_event)
|
|
4350
4529
|
.with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-elasticsearch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.2.
|
|
4
|
+
version: 4.2.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- diogo
|
|
@@ -10,7 +10,7 @@ authors:
|
|
|
10
10
|
autorequire:
|
|
11
11
|
bindir: bin
|
|
12
12
|
cert_chain: []
|
|
13
|
-
date: 2020-10-
|
|
13
|
+
date: 2020-10-16 00:00:00.000000000 Z
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
16
16
|
name: fluentd
|