fluent-plugin-elasticsearch 4.0.7 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/History.md +25 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +78 -3
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +68 -31
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_fallback_selector.rb +73 -0
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +408 -5
- data/test/plugin/test_out_elasticsearch_dynamic.rb +21 -3
- metadata +10 -2
|
@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
|
|
|
62
62
|
d = driver('')
|
|
63
63
|
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
|
64
64
|
if @use_tls_minmax_version
|
|
65
|
-
assert_equal({max_version: OpenSSL::SSL::
|
|
66
|
-
min_version: OpenSSL::SSL::
|
|
65
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
|
66
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
|
67
67
|
else
|
|
68
68
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
|
69
69
|
end
|
|
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
|
18
18
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
+
test "invalid configuration" do
|
|
22
|
+
assert_raise(Fluent::ConfigError) do
|
|
23
|
+
create_driver("use_record_as_seed true")
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
21
27
|
def sample_record
|
|
22
28
|
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
|
23
29
|
end
|
|
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
|
41
47
|
assert_equal(Base64.strict_encode64(SecureRandom.uuid),
|
|
42
48
|
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
43
49
|
end
|
|
50
|
+
|
|
51
|
+
class UseRecordAsSeedTest < self
|
|
52
|
+
data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
|
|
53
|
+
"sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
|
|
54
|
+
"sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
|
|
55
|
+
"sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
|
|
56
|
+
)
|
|
57
|
+
def test_simple(data)
|
|
58
|
+
hash_type, expected = data
|
|
59
|
+
d = create_driver(%[
|
|
60
|
+
use_record_as_seed true
|
|
61
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
62
|
+
hash_type #{hash_type}
|
|
63
|
+
])
|
|
64
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
65
|
+
d.run(default_tag: 'test') do
|
|
66
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
67
|
+
end
|
|
68
|
+
assert_equal(expected,
|
|
69
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
|
|
73
|
+
"sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
|
|
74
|
+
"sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
|
|
75
|
+
"sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
|
|
76
|
+
def test_record_with_tag(data)
|
|
77
|
+
hash_type, expected = data
|
|
78
|
+
d = create_driver(%[
|
|
79
|
+
use_record_as_seed true
|
|
80
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
81
|
+
hash_type #{hash_type}
|
|
82
|
+
include_tag_in_seed true
|
|
83
|
+
])
|
|
84
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
85
|
+
d.run(default_tag: 'test.fluentd') do
|
|
86
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
87
|
+
end
|
|
88
|
+
assert_equal(expected,
|
|
89
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
|
|
93
|
+
"sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
|
|
94
|
+
"sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
|
|
95
|
+
"sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
|
|
96
|
+
def test_record_with_time(data)
|
|
97
|
+
hash_type, expected = data
|
|
98
|
+
d = create_driver(%[
|
|
99
|
+
use_record_as_seed true
|
|
100
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
101
|
+
hash_type #{hash_type}
|
|
102
|
+
include_time_in_seed true
|
|
103
|
+
])
|
|
104
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
105
|
+
d.run(default_tag: 'test.fluentd') do
|
|
106
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
107
|
+
end
|
|
108
|
+
assert_equal(expected,
|
|
109
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
|
|
113
|
+
"sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
|
|
114
|
+
"sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
|
|
115
|
+
"sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
|
|
116
|
+
def test_record_with_tag_and_time
|
|
117
|
+
hash_type, expected = data
|
|
118
|
+
d = create_driver(%[
|
|
119
|
+
use_record_as_seed true
|
|
120
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
121
|
+
hash_type #{hash_type}
|
|
122
|
+
include_tag_in_seed true
|
|
123
|
+
include_time_in_seed true
|
|
124
|
+
])
|
|
125
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
126
|
+
d.run(default_tag: 'test.fluentd') do
|
|
127
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
128
|
+
end
|
|
129
|
+
assert_equal(expected,
|
|
130
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
class UseEntireRecordAsSeedTest < self
|
|
135
|
+
data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
|
|
136
|
+
"sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
|
|
137
|
+
"sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
|
|
138
|
+
"sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
|
|
139
|
+
def test_record
|
|
140
|
+
hash_type, expected = data
|
|
141
|
+
d = create_driver(%[
|
|
142
|
+
use_record_as_seed true
|
|
143
|
+
use_entire_record true
|
|
144
|
+
hash_type #{hash_type}
|
|
145
|
+
])
|
|
146
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
147
|
+
d.run(default_tag: 'test.fluentd') do
|
|
148
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
149
|
+
end
|
|
150
|
+
assert_equal(expected,
|
|
151
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
|
|
155
|
+
"sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
|
|
156
|
+
"sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
|
|
157
|
+
"sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
|
|
158
|
+
def test_record_with_tag
|
|
159
|
+
hash_type, expected = data
|
|
160
|
+
d = create_driver(%[
|
|
161
|
+
use_record_as_seed true
|
|
162
|
+
use_entire_record true
|
|
163
|
+
hash_type #{hash_type}
|
|
164
|
+
include_tag_in_seed true
|
|
165
|
+
])
|
|
166
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
167
|
+
d.run(default_tag: 'test.fluentd') do
|
|
168
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
169
|
+
end
|
|
170
|
+
assert_equal(expected,
|
|
171
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
|
|
175
|
+
"sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
|
|
176
|
+
"sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
|
|
177
|
+
"sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
|
|
178
|
+
def test_record_with_time
|
|
179
|
+
hash_type, expected = data
|
|
180
|
+
d = create_driver(%[
|
|
181
|
+
use_record_as_seed true
|
|
182
|
+
use_entire_record true
|
|
183
|
+
hash_type #{hash_type}
|
|
184
|
+
include_time_in_seed true
|
|
185
|
+
])
|
|
186
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
187
|
+
d.run(default_tag: 'test.fluentd') do
|
|
188
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
189
|
+
end
|
|
190
|
+
assert_equal(expected,
|
|
191
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
|
|
195
|
+
"sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
|
|
196
|
+
"sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
|
|
197
|
+
"sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
|
|
198
|
+
def test_record_with_tag_and_time
|
|
199
|
+
hash_type, expected = data
|
|
200
|
+
d = create_driver(%[
|
|
201
|
+
use_record_as_seed true
|
|
202
|
+
use_entire_record true
|
|
203
|
+
hash_type #{hash_type}
|
|
204
|
+
include_tag_in_seed true
|
|
205
|
+
include_time_in_seed true
|
|
206
|
+
])
|
|
207
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
208
|
+
d.run(default_tag: 'test.fluentd') do
|
|
209
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
210
|
+
end
|
|
211
|
+
assert_equal(expected,
|
|
212
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
213
|
+
end
|
|
214
|
+
end
|
|
44
215
|
end
|
|
@@ -230,7 +230,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
230
230
|
assert_nil instance.ssl_max_version
|
|
231
231
|
assert_nil instance.ssl_min_version
|
|
232
232
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
|
233
|
-
assert_equal({max_version: OpenSSL::SSL::
|
|
233
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
|
234
234
|
instance.ssl_version_options)
|
|
235
235
|
else
|
|
236
236
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
|
@@ -284,7 +284,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
284
284
|
}
|
|
285
285
|
instance = driver(config).instance
|
|
286
286
|
|
|
287
|
-
assert_equal
|
|
287
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
|
288
|
+
|
|
289
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
|
290
|
+
to_return(status: 200, body: "", headers: {})
|
|
291
|
+
driver.run(default_tag: 'test') do
|
|
292
|
+
driver.feed(sample_record)
|
|
293
|
+
end
|
|
294
|
+
compressable = instance.compressable_connection
|
|
295
|
+
|
|
296
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
|
288
297
|
end
|
|
289
298
|
|
|
290
299
|
test 'check compression option is passed to transport' do
|
|
@@ -295,7 +304,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
295
304
|
}
|
|
296
305
|
instance = driver(config).instance
|
|
297
306
|
|
|
298
|
-
assert_equal
|
|
307
|
+
assert_equal false, instance.client.transport.options[:compression]
|
|
308
|
+
|
|
309
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
|
310
|
+
to_return(status: 200, body: "", headers: {})
|
|
311
|
+
driver.run(default_tag: 'test') do
|
|
312
|
+
driver.feed(sample_record)
|
|
313
|
+
end
|
|
314
|
+
compressable = instance.compressable_connection
|
|
315
|
+
|
|
316
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
|
299
317
|
end
|
|
300
318
|
|
|
301
319
|
test 'configure Content-Type' do
|
|
@@ -1035,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1035
1053
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1036
1054
|
end
|
|
1037
1055
|
|
|
1056
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
|
|
1057
|
+
cwd = File.dirname(__FILE__)
|
|
1058
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1059
|
+
|
|
1060
|
+
config = %{
|
|
1061
|
+
host logs.google.com
|
|
1062
|
+
port 777
|
|
1063
|
+
scheme https
|
|
1064
|
+
path /es/
|
|
1065
|
+
user john
|
|
1066
|
+
password doe
|
|
1067
|
+
template_name logstash
|
|
1068
|
+
template_file #{template_file}
|
|
1069
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1070
|
+
enable_ilm true
|
|
1071
|
+
logstash_format true
|
|
1072
|
+
application_name log
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
date_str = Time.now.strftime("%Y.%m.%d")
|
|
1076
|
+
# connection start
|
|
1077
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1078
|
+
with(basic_auth: ['john', 'doe']).
|
|
1079
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1080
|
+
# check if template exists
|
|
1081
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1082
|
+
with(basic_auth: ['john', 'doe']).
|
|
1083
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1084
|
+
# creation
|
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1086
|
+
with(basic_auth: ['john', 'doe']).
|
|
1087
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1088
|
+
# check if alias exists
|
|
1089
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-log-#{date_str}").
|
|
1090
|
+
with(basic_auth: ['john', 'doe']).
|
|
1091
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1092
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1093
|
+
with(basic_auth: ['john', 'doe']).
|
|
1094
|
+
to_return(status: 404, body: "", headers: {})
|
|
1095
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1096
|
+
with(basic_auth: ['john', 'doe'],
|
|
1097
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
|
1098
|
+
to_return(status: 200, body: "", headers: {})
|
|
1099
|
+
# put the alias for the index
|
|
1100
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1101
|
+
with(basic_auth: ['john', 'doe']).
|
|
1102
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1103
|
+
|
|
1104
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-log-#{date_str}").
|
|
1105
|
+
with(basic_auth: ['john', 'doe'],
|
|
1106
|
+
body: "{\"aliases\":{\"logstash-log-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1107
|
+
to_return(status: 200, body: "", headers: {})
|
|
1108
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1109
|
+
with(basic_auth: ['john', 'doe']).
|
|
1110
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1111
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1112
|
+
with(basic_auth: ['john', 'doe']).
|
|
1113
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1114
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1115
|
+
with(basic_auth: ['john', 'doe'],
|
|
1116
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1117
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1118
|
+
|
|
1119
|
+
driver(config)
|
|
1120
|
+
|
|
1121
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1122
|
+
driver.run(default_tag: 'test') do
|
|
1123
|
+
driver.feed(sample_record)
|
|
1124
|
+
end
|
|
1125
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}", times: 1)
|
|
1126
|
+
|
|
1127
|
+
assert_requested(elastic_request)
|
|
1128
|
+
end
|
|
1129
|
+
|
|
1038
1130
|
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
|
|
1039
1131
|
cwd = File.dirname(__FILE__)
|
|
1040
1132
|
template_file = File.join(cwd, 'test_template.json')
|
|
@@ -1299,6 +1391,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1299
1391
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1300
1392
|
end
|
|
1301
1393
|
|
|
1394
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
|
1395
|
+
cwd = File.dirname(__FILE__)
|
|
1396
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1397
|
+
|
|
1398
|
+
config = %{
|
|
1399
|
+
host logs.google.com
|
|
1400
|
+
port 777
|
|
1401
|
+
scheme https
|
|
1402
|
+
path /es/
|
|
1403
|
+
user john
|
|
1404
|
+
password doe
|
|
1405
|
+
template_name logstash
|
|
1406
|
+
template_file #{template_file}
|
|
1407
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1408
|
+
ilm_policy_id fluentd-policy
|
|
1409
|
+
enable_ilm true
|
|
1410
|
+
index_name logstash
|
|
1411
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
# connection start
|
|
1415
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1416
|
+
with(basic_auth: ['john', 'doe']).
|
|
1417
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1418
|
+
# check if template exists
|
|
1419
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1420
|
+
with(basic_auth: ['john', 'doe']).
|
|
1421
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1422
|
+
# creation
|
|
1423
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1424
|
+
with(basic_auth: ['john', 'doe']).
|
|
1425
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1426
|
+
# check if alias exists
|
|
1427
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1428
|
+
with(basic_auth: ['john', 'doe']).
|
|
1429
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1430
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1431
|
+
with(basic_auth: ['john', 'doe']).
|
|
1432
|
+
to_return(status: 404, body: "", headers: {})
|
|
1433
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1434
|
+
with(basic_auth: ['john', 'doe'],
|
|
1435
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1436
|
+
to_return(status: 200, body: "", headers: {})
|
|
1437
|
+
# put the alias for the index
|
|
1438
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1439
|
+
with(basic_auth: ['john', 'doe']).
|
|
1440
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1441
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1442
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1443
|
+
to_return(status: 200, body: "", headers: {})
|
|
1444
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1445
|
+
with(basic_auth: ['john', 'doe']).
|
|
1446
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1447
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1448
|
+
with(basic_auth: ['john', 'doe']).
|
|
1449
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1450
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1451
|
+
with(basic_auth: ['john', 'doe'],
|
|
1452
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1453
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1454
|
+
|
|
1455
|
+
driver(config)
|
|
1456
|
+
|
|
1457
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1458
|
+
driver.run(default_tag: 'test') do
|
|
1459
|
+
driver.feed(sample_record)
|
|
1460
|
+
end
|
|
1461
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1462
|
+
|
|
1463
|
+
assert_requested(elastic_request)
|
|
1464
|
+
end
|
|
1465
|
+
|
|
1466
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
|
1467
|
+
def test_tag_placeholder
|
|
1468
|
+
cwd = File.dirname(__FILE__)
|
|
1469
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1470
|
+
|
|
1471
|
+
config = %{
|
|
1472
|
+
host logs.google.com
|
|
1473
|
+
port 777
|
|
1474
|
+
scheme https
|
|
1475
|
+
path /es/
|
|
1476
|
+
user john
|
|
1477
|
+
password doe
|
|
1478
|
+
template_name logstash
|
|
1479
|
+
template_file #{template_file}
|
|
1480
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1481
|
+
ilm_policy_id ${tag}
|
|
1482
|
+
enable_ilm true
|
|
1483
|
+
index_name logstash
|
|
1484
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
# connection start
|
|
1488
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1489
|
+
with(basic_auth: ['john', 'doe']).
|
|
1490
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1491
|
+
# check if template exists
|
|
1492
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1493
|
+
with(basic_auth: ['john', 'doe']).
|
|
1494
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1495
|
+
# creation
|
|
1496
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1497
|
+
with(basic_auth: ['john', 'doe']).
|
|
1498
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1499
|
+
# check if alias exists
|
|
1500
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1501
|
+
with(basic_auth: ['john', 'doe']).
|
|
1502
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1503
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1504
|
+
with(basic_auth: ['john', 'doe']).
|
|
1505
|
+
to_return(status: 404, body: "", headers: {})
|
|
1506
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1507
|
+
with(basic_auth: ['john', 'doe'],
|
|
1508
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1509
|
+
to_return(status: 200, body: "", headers: {})
|
|
1510
|
+
# put the alias for the index
|
|
1511
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1512
|
+
with(basic_auth: ['john', 'doe']).
|
|
1513
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1514
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1515
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1516
|
+
to_return(status: 200, body: "", headers: {})
|
|
1517
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1518
|
+
with(basic_auth: ['john', 'doe']).
|
|
1519
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1520
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1521
|
+
with(basic_auth: ['john', 'doe']).
|
|
1522
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1523
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1524
|
+
with(basic_auth: ['john', 'doe'],
|
|
1525
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1526
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1527
|
+
|
|
1528
|
+
driver(config)
|
|
1529
|
+
|
|
1530
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1531
|
+
driver.run(default_tag: 'fluentd-policy') do
|
|
1532
|
+
driver.feed(sample_record)
|
|
1533
|
+
end
|
|
1534
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1535
|
+
|
|
1536
|
+
assert_requested(elastic_request)
|
|
1537
|
+
end
|
|
1538
|
+
|
|
1539
|
+
def test_tag_placeholder_with_multiple_policies
|
|
1540
|
+
cwd = File.dirname(__FILE__)
|
|
1541
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1542
|
+
|
|
1543
|
+
config = %{
|
|
1544
|
+
host logs.google.com
|
|
1545
|
+
port 777
|
|
1546
|
+
scheme https
|
|
1547
|
+
path /es/
|
|
1548
|
+
user john
|
|
1549
|
+
password doe
|
|
1550
|
+
template_name logstash
|
|
1551
|
+
template_file #{template_file}
|
|
1552
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1553
|
+
ilm_policy_id ${tag}
|
|
1554
|
+
enable_ilm true
|
|
1555
|
+
index_name logstash
|
|
1556
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
# connection start
|
|
1560
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1561
|
+
with(basic_auth: ['john', 'doe']).
|
|
1562
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1563
|
+
# check if template exists
|
|
1564
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1565
|
+
with(basic_auth: ['john', 'doe']).
|
|
1566
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1567
|
+
# creation
|
|
1568
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1569
|
+
with(basic_auth: ['john', 'doe']).
|
|
1570
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1571
|
+
# check if alias exists
|
|
1572
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1573
|
+
with(basic_auth: ['john', 'doe']).
|
|
1574
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1575
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1576
|
+
with(basic_auth: ['john', 'doe']).
|
|
1577
|
+
to_return(status: 404, body: "", headers: {})
|
|
1578
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1579
|
+
with(basic_auth: ['john', 'doe'],
|
|
1580
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1581
|
+
to_return(status: 200, body: "", headers: {})
|
|
1582
|
+
# put the alias for the index
|
|
1583
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1584
|
+
with(basic_auth: ['john', 'doe']).
|
|
1585
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1586
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1587
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1588
|
+
to_return(status: 200, body: "", headers: {})
|
|
1589
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1590
|
+
with(basic_auth: ['john', 'doe']).
|
|
1591
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1592
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1593
|
+
with(basic_auth: ['john', 'doe']).
|
|
1594
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1595
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1596
|
+
with(basic_auth: ['john', 'doe'],
|
|
1597
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
|
1598
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1599
|
+
|
|
1600
|
+
driver(config)
|
|
1601
|
+
|
|
1602
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1603
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
|
1604
|
+
driver.feed(sample_record)
|
|
1605
|
+
end
|
|
1606
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1607
|
+
|
|
1608
|
+
assert_requested(elastic_request)
|
|
1609
|
+
end
|
|
1610
|
+
end
|
|
1611
|
+
|
|
1302
1612
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
|
1303
1613
|
cwd = File.dirname(__FILE__)
|
|
1304
1614
|
template_file = File.join(cwd, 'test_template.json')
|
|
@@ -1371,6 +1681,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1371
1681
|
|
|
1372
1682
|
assert_requested(elastic_request)
|
|
1373
1683
|
end
|
|
1684
|
+
|
|
1685
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
|
|
1686
|
+
cwd = File.dirname(__FILE__)
|
|
1687
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1688
|
+
|
|
1689
|
+
config = Fluent::Config::Element.new(
|
|
1690
|
+
'ROOT', '', {
|
|
1691
|
+
'@type' => 'elasticsearch',
|
|
1692
|
+
'host' => 'logs.google.com',
|
|
1693
|
+
'port' => 777,
|
|
1694
|
+
'scheme' => "https",
|
|
1695
|
+
'path' => "/es/",
|
|
1696
|
+
'user' => 'john',
|
|
1697
|
+
'password' => 'doe',
|
|
1698
|
+
'template_name' => 'logstash',
|
|
1699
|
+
'template_file' => "#{template_file}",
|
|
1700
|
+
'index_date_pattern' => 'now/w{xxxx.ww}',
|
|
1701
|
+
'index_name' => "${taskDef}-%Y.%m",
|
|
1702
|
+
'enable_ilm' => true,
|
|
1703
|
+
}, [
|
|
1704
|
+
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
|
1705
|
+
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
|
1706
|
+
'timekey' => 3600,
|
|
1707
|
+
}, [])
|
|
1708
|
+
]
|
|
1709
|
+
)
|
|
1710
|
+
|
|
1711
|
+
task_def_value = "task_definition"
|
|
1712
|
+
date_str = Time.now.strftime("%Y.%m")
|
|
1713
|
+
# connection start
|
|
1714
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1715
|
+
with(basic_auth: ['john', 'doe']).
|
|
1716
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1717
|
+
# check if template exists
|
|
1718
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1719
|
+
with(basic_auth: ['john', 'doe']).
|
|
1720
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1721
|
+
# creation
|
|
1722
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1723
|
+
with(basic_auth: ['john', 'doe'],
|
|
1724
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
|
1725
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1726
|
+
# check if alias exists
|
|
1727
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
|
1728
|
+
with(basic_auth: ['john', 'doe']).
|
|
1729
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1730
|
+
# put the alias for the index
|
|
1731
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1732
|
+
with(basic_auth: ['john', 'doe']).
|
|
1733
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1734
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
|
|
1735
|
+
with(basic_auth: ['john', 'doe'],
|
|
1736
|
+
:body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1737
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1738
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1739
|
+
with(basic_auth: ['john', 'doe']).
|
|
1740
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1741
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1742
|
+
with(basic_auth: ['john', 'doe']).
|
|
1743
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1744
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1745
|
+
with(basic_auth: ['john', 'doe'],
|
|
1746
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1747
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1748
|
+
|
|
1749
|
+
driver(config)
|
|
1750
|
+
|
|
1751
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1752
|
+
driver.run(default_tag: 'test') do
|
|
1753
|
+
driver.feed(sample_record.merge("taskDef" => task_def_value))
|
|
1754
|
+
end
|
|
1755
|
+
assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
|
|
1756
|
+
|
|
1757
|
+
assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
|
|
1758
|
+
|
|
1759
|
+
assert_requested(elastic_request)
|
|
1760
|
+
end
|
|
1374
1761
|
end
|
|
1375
1762
|
|
|
1376
1763
|
def test_custom_template_create
|
|
@@ -1627,11 +2014,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1627
2014
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1628
2015
|
# check if alias exists
|
|
1629
2016
|
timestr = Time.now.strftime("%Y.%m.%d")
|
|
1630
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
|
2017
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
|
|
1631
2018
|
with(basic_auth: ['john', 'doe']).
|
|
1632
2019
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1633
2020
|
# put the alias for the index
|
|
1634
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
|
2021
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-myapp-#{timestr}").
|
|
1635
2022
|
with(basic_auth: ['john', 'doe']).
|
|
1636
2023
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1637
2024
|
|
|
@@ -2911,6 +3298,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2911
3298
|
|
|
2912
3299
|
data("border" => {"es_version" => 6, "_type" => "mytype"},
|
|
2913
3300
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3301
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2914
3302
|
)
|
|
2915
3303
|
def test_writes_to_speficied_type(data)
|
|
2916
3304
|
driver('', data["es_version"]).configure("type_name mytype\n")
|
|
@@ -2923,6 +3311,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2923
3311
|
|
|
2924
3312
|
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
2925
3313
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3314
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2926
3315
|
)
|
|
2927
3316
|
def test_writes_to_speficied_type_with_placeholders(data)
|
|
2928
3317
|
driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
|
|
@@ -2933,6 +3322,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2933
3322
|
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
2934
3323
|
end
|
|
2935
3324
|
|
|
3325
|
+
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
3326
|
+
"fixed_behavior"=> {"es_version" => 7, "_type" => nil},
|
|
3327
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
3328
|
+
)
|
|
3329
|
+
def test_writes_to_speficied_type_with_suppress_type_name(data)
|
|
3330
|
+
driver('', data["es_version"])
|
|
3331
|
+
.configure("type_name mytype.${tag}\nsuppress_type_name true")
|
|
3332
|
+
stub_elastic
|
|
3333
|
+
driver.run(default_tag: 'test') do
|
|
3334
|
+
driver.feed(sample_record)
|
|
3335
|
+
end
|
|
3336
|
+
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
3337
|
+
end
|
|
3338
|
+
|
|
2936
3339
|
data("old" => {"es_version" => 2, "_type" => "local-override"},
|
|
2937
3340
|
"old_behavior" => {"es_version" => 5, "_type" => "local-override"},
|
|
2938
3341
|
"border" => {"es_version" => 6, "_type" => "fluentd"},
|