fluent-plugin-elasticsearch 4.0.7 → 4.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/History.md +25 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +78 -3
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +68 -31
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_fallback_selector.rb +73 -0
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +408 -5
- data/test/plugin/test_out_elasticsearch_dynamic.rb +21 -3
- metadata +10 -2
|
@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
|
|
|
62
62
|
d = driver('')
|
|
63
63
|
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
|
64
64
|
if @use_tls_minmax_version
|
|
65
|
-
assert_equal({max_version: OpenSSL::SSL::
|
|
66
|
-
min_version: OpenSSL::SSL::
|
|
65
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
|
66
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
|
67
67
|
else
|
|
68
68
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
|
69
69
|
end
|
|
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
|
18
18
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
+
test "invalid configuration" do
|
|
22
|
+
assert_raise(Fluent::ConfigError) do
|
|
23
|
+
create_driver("use_record_as_seed true")
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
21
27
|
def sample_record
|
|
22
28
|
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
|
23
29
|
end
|
|
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
|
41
47
|
assert_equal(Base64.strict_encode64(SecureRandom.uuid),
|
|
42
48
|
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
43
49
|
end
|
|
50
|
+
|
|
51
|
+
class UseRecordAsSeedTest < self
|
|
52
|
+
data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
|
|
53
|
+
"sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
|
|
54
|
+
"sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
|
|
55
|
+
"sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
|
|
56
|
+
)
|
|
57
|
+
def test_simple(data)
|
|
58
|
+
hash_type, expected = data
|
|
59
|
+
d = create_driver(%[
|
|
60
|
+
use_record_as_seed true
|
|
61
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
62
|
+
hash_type #{hash_type}
|
|
63
|
+
])
|
|
64
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
65
|
+
d.run(default_tag: 'test') do
|
|
66
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
67
|
+
end
|
|
68
|
+
assert_equal(expected,
|
|
69
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
|
|
73
|
+
"sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
|
|
74
|
+
"sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
|
|
75
|
+
"sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
|
|
76
|
+
def test_record_with_tag(data)
|
|
77
|
+
hash_type, expected = data
|
|
78
|
+
d = create_driver(%[
|
|
79
|
+
use_record_as_seed true
|
|
80
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
81
|
+
hash_type #{hash_type}
|
|
82
|
+
include_tag_in_seed true
|
|
83
|
+
])
|
|
84
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
85
|
+
d.run(default_tag: 'test.fluentd') do
|
|
86
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
87
|
+
end
|
|
88
|
+
assert_equal(expected,
|
|
89
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
|
|
93
|
+
"sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
|
|
94
|
+
"sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
|
|
95
|
+
"sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
|
|
96
|
+
def test_record_with_time(data)
|
|
97
|
+
hash_type, expected = data
|
|
98
|
+
d = create_driver(%[
|
|
99
|
+
use_record_as_seed true
|
|
100
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
101
|
+
hash_type #{hash_type}
|
|
102
|
+
include_time_in_seed true
|
|
103
|
+
])
|
|
104
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
105
|
+
d.run(default_tag: 'test.fluentd') do
|
|
106
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
107
|
+
end
|
|
108
|
+
assert_equal(expected,
|
|
109
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
|
|
113
|
+
"sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
|
|
114
|
+
"sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
|
|
115
|
+
"sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
|
|
116
|
+
def test_record_with_tag_and_time
|
|
117
|
+
hash_type, expected = data
|
|
118
|
+
d = create_driver(%[
|
|
119
|
+
use_record_as_seed true
|
|
120
|
+
record_keys age,parent_id,routing_id,custom_key
|
|
121
|
+
hash_type #{hash_type}
|
|
122
|
+
include_tag_in_seed true
|
|
123
|
+
include_time_in_seed true
|
|
124
|
+
])
|
|
125
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
126
|
+
d.run(default_tag: 'test.fluentd') do
|
|
127
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
128
|
+
end
|
|
129
|
+
assert_equal(expected,
|
|
130
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
class UseEntireRecordAsSeedTest < self
|
|
135
|
+
data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
|
|
136
|
+
"sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
|
|
137
|
+
"sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
|
|
138
|
+
"sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
|
|
139
|
+
def test_record
|
|
140
|
+
hash_type, expected = data
|
|
141
|
+
d = create_driver(%[
|
|
142
|
+
use_record_as_seed true
|
|
143
|
+
use_entire_record true
|
|
144
|
+
hash_type #{hash_type}
|
|
145
|
+
])
|
|
146
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
147
|
+
d.run(default_tag: 'test.fluentd') do
|
|
148
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
149
|
+
end
|
|
150
|
+
assert_equal(expected,
|
|
151
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
|
|
155
|
+
"sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
|
|
156
|
+
"sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
|
|
157
|
+
"sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
|
|
158
|
+
def test_record_with_tag
|
|
159
|
+
hash_type, expected = data
|
|
160
|
+
d = create_driver(%[
|
|
161
|
+
use_record_as_seed true
|
|
162
|
+
use_entire_record true
|
|
163
|
+
hash_type #{hash_type}
|
|
164
|
+
include_tag_in_seed true
|
|
165
|
+
])
|
|
166
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
167
|
+
d.run(default_tag: 'test.fluentd') do
|
|
168
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
169
|
+
end
|
|
170
|
+
assert_equal(expected,
|
|
171
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
|
|
175
|
+
"sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
|
|
176
|
+
"sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
|
|
177
|
+
"sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
|
|
178
|
+
def test_record_with_time
|
|
179
|
+
hash_type, expected = data
|
|
180
|
+
d = create_driver(%[
|
|
181
|
+
use_record_as_seed true
|
|
182
|
+
use_entire_record true
|
|
183
|
+
hash_type #{hash_type}
|
|
184
|
+
include_time_in_seed true
|
|
185
|
+
])
|
|
186
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
187
|
+
d.run(default_tag: 'test.fluentd') do
|
|
188
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
189
|
+
end
|
|
190
|
+
assert_equal(expected,
|
|
191
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
|
|
195
|
+
"sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
|
|
196
|
+
"sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
|
|
197
|
+
"sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
|
|
198
|
+
def test_record_with_tag_and_time
|
|
199
|
+
hash_type, expected = data
|
|
200
|
+
d = create_driver(%[
|
|
201
|
+
use_record_as_seed true
|
|
202
|
+
use_entire_record true
|
|
203
|
+
hash_type #{hash_type}
|
|
204
|
+
include_tag_in_seed true
|
|
205
|
+
include_time_in_seed true
|
|
206
|
+
])
|
|
207
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
|
208
|
+
d.run(default_tag: 'test.fluentd') do
|
|
209
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
|
210
|
+
end
|
|
211
|
+
assert_equal(expected,
|
|
212
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
|
213
|
+
end
|
|
214
|
+
end
|
|
44
215
|
end
|
|
@@ -230,7 +230,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
230
230
|
assert_nil instance.ssl_max_version
|
|
231
231
|
assert_nil instance.ssl_min_version
|
|
232
232
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
|
233
|
-
assert_equal({max_version: OpenSSL::SSL::
|
|
233
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
|
234
234
|
instance.ssl_version_options)
|
|
235
235
|
else
|
|
236
236
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
|
@@ -284,7 +284,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
284
284
|
}
|
|
285
285
|
instance = driver(config).instance
|
|
286
286
|
|
|
287
|
-
assert_equal
|
|
287
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
|
288
|
+
|
|
289
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
|
290
|
+
to_return(status: 200, body: "", headers: {})
|
|
291
|
+
driver.run(default_tag: 'test') do
|
|
292
|
+
driver.feed(sample_record)
|
|
293
|
+
end
|
|
294
|
+
compressable = instance.compressable_connection
|
|
295
|
+
|
|
296
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
|
288
297
|
end
|
|
289
298
|
|
|
290
299
|
test 'check compression option is passed to transport' do
|
|
@@ -295,7 +304,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
295
304
|
}
|
|
296
305
|
instance = driver(config).instance
|
|
297
306
|
|
|
298
|
-
assert_equal
|
|
307
|
+
assert_equal false, instance.client.transport.options[:compression]
|
|
308
|
+
|
|
309
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
|
310
|
+
to_return(status: 200, body: "", headers: {})
|
|
311
|
+
driver.run(default_tag: 'test') do
|
|
312
|
+
driver.feed(sample_record)
|
|
313
|
+
end
|
|
314
|
+
compressable = instance.compressable_connection
|
|
315
|
+
|
|
316
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
|
299
317
|
end
|
|
300
318
|
|
|
301
319
|
test 'configure Content-Type' do
|
|
@@ -1035,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1035
1053
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1036
1054
|
end
|
|
1037
1055
|
|
|
1056
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
|
|
1057
|
+
cwd = File.dirname(__FILE__)
|
|
1058
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1059
|
+
|
|
1060
|
+
config = %{
|
|
1061
|
+
host logs.google.com
|
|
1062
|
+
port 777
|
|
1063
|
+
scheme https
|
|
1064
|
+
path /es/
|
|
1065
|
+
user john
|
|
1066
|
+
password doe
|
|
1067
|
+
template_name logstash
|
|
1068
|
+
template_file #{template_file}
|
|
1069
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1070
|
+
enable_ilm true
|
|
1071
|
+
logstash_format true
|
|
1072
|
+
application_name log
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
date_str = Time.now.strftime("%Y.%m.%d")
|
|
1076
|
+
# connection start
|
|
1077
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1078
|
+
with(basic_auth: ['john', 'doe']).
|
|
1079
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1080
|
+
# check if template exists
|
|
1081
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1082
|
+
with(basic_auth: ['john', 'doe']).
|
|
1083
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1084
|
+
# creation
|
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1086
|
+
with(basic_auth: ['john', 'doe']).
|
|
1087
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1088
|
+
# check if alias exists
|
|
1089
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-log-#{date_str}").
|
|
1090
|
+
with(basic_auth: ['john', 'doe']).
|
|
1091
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1092
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1093
|
+
with(basic_auth: ['john', 'doe']).
|
|
1094
|
+
to_return(status: 404, body: "", headers: {})
|
|
1095
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
|
1096
|
+
with(basic_auth: ['john', 'doe'],
|
|
1097
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
|
1098
|
+
to_return(status: 200, body: "", headers: {})
|
|
1099
|
+
# put the alias for the index
|
|
1100
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1101
|
+
with(basic_auth: ['john', 'doe']).
|
|
1102
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1103
|
+
|
|
1104
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-log-#{date_str}").
|
|
1105
|
+
with(basic_auth: ['john', 'doe'],
|
|
1106
|
+
body: "{\"aliases\":{\"logstash-log-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1107
|
+
to_return(status: 200, body: "", headers: {})
|
|
1108
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1109
|
+
with(basic_auth: ['john', 'doe']).
|
|
1110
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1111
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1112
|
+
with(basic_auth: ['john', 'doe']).
|
|
1113
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1114
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1115
|
+
with(basic_auth: ['john', 'doe'],
|
|
1116
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1117
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1118
|
+
|
|
1119
|
+
driver(config)
|
|
1120
|
+
|
|
1121
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1122
|
+
driver.run(default_tag: 'test') do
|
|
1123
|
+
driver.feed(sample_record)
|
|
1124
|
+
end
|
|
1125
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}", times: 1)
|
|
1126
|
+
|
|
1127
|
+
assert_requested(elastic_request)
|
|
1128
|
+
end
|
|
1129
|
+
|
|
1038
1130
|
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
|
|
1039
1131
|
cwd = File.dirname(__FILE__)
|
|
1040
1132
|
template_file = File.join(cwd, 'test_template.json')
|
|
@@ -1299,6 +1391,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1299
1391
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1300
1392
|
end
|
|
1301
1393
|
|
|
1394
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
|
1395
|
+
cwd = File.dirname(__FILE__)
|
|
1396
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1397
|
+
|
|
1398
|
+
config = %{
|
|
1399
|
+
host logs.google.com
|
|
1400
|
+
port 777
|
|
1401
|
+
scheme https
|
|
1402
|
+
path /es/
|
|
1403
|
+
user john
|
|
1404
|
+
password doe
|
|
1405
|
+
template_name logstash
|
|
1406
|
+
template_file #{template_file}
|
|
1407
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1408
|
+
ilm_policy_id fluentd-policy
|
|
1409
|
+
enable_ilm true
|
|
1410
|
+
index_name logstash
|
|
1411
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
# connection start
|
|
1415
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1416
|
+
with(basic_auth: ['john', 'doe']).
|
|
1417
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1418
|
+
# check if template exists
|
|
1419
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1420
|
+
with(basic_auth: ['john', 'doe']).
|
|
1421
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1422
|
+
# creation
|
|
1423
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1424
|
+
with(basic_auth: ['john', 'doe']).
|
|
1425
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1426
|
+
# check if alias exists
|
|
1427
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1428
|
+
with(basic_auth: ['john', 'doe']).
|
|
1429
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1430
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1431
|
+
with(basic_auth: ['john', 'doe']).
|
|
1432
|
+
to_return(status: 404, body: "", headers: {})
|
|
1433
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1434
|
+
with(basic_auth: ['john', 'doe'],
|
|
1435
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1436
|
+
to_return(status: 200, body: "", headers: {})
|
|
1437
|
+
# put the alias for the index
|
|
1438
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1439
|
+
with(basic_auth: ['john', 'doe']).
|
|
1440
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1441
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1442
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1443
|
+
to_return(status: 200, body: "", headers: {})
|
|
1444
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1445
|
+
with(basic_auth: ['john', 'doe']).
|
|
1446
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1447
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1448
|
+
with(basic_auth: ['john', 'doe']).
|
|
1449
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1450
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1451
|
+
with(basic_auth: ['john', 'doe'],
|
|
1452
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1453
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1454
|
+
|
|
1455
|
+
driver(config)
|
|
1456
|
+
|
|
1457
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1458
|
+
driver.run(default_tag: 'test') do
|
|
1459
|
+
driver.feed(sample_record)
|
|
1460
|
+
end
|
|
1461
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1462
|
+
|
|
1463
|
+
assert_requested(elastic_request)
|
|
1464
|
+
end
|
|
1465
|
+
|
|
1466
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
|
1467
|
+
def test_tag_placeholder
|
|
1468
|
+
cwd = File.dirname(__FILE__)
|
|
1469
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1470
|
+
|
|
1471
|
+
config = %{
|
|
1472
|
+
host logs.google.com
|
|
1473
|
+
port 777
|
|
1474
|
+
scheme https
|
|
1475
|
+
path /es/
|
|
1476
|
+
user john
|
|
1477
|
+
password doe
|
|
1478
|
+
template_name logstash
|
|
1479
|
+
template_file #{template_file}
|
|
1480
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1481
|
+
ilm_policy_id ${tag}
|
|
1482
|
+
enable_ilm true
|
|
1483
|
+
index_name logstash
|
|
1484
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
# connection start
|
|
1488
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1489
|
+
with(basic_auth: ['john', 'doe']).
|
|
1490
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1491
|
+
# check if template exists
|
|
1492
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1493
|
+
with(basic_auth: ['john', 'doe']).
|
|
1494
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1495
|
+
# creation
|
|
1496
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1497
|
+
with(basic_auth: ['john', 'doe']).
|
|
1498
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1499
|
+
# check if alias exists
|
|
1500
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1501
|
+
with(basic_auth: ['john', 'doe']).
|
|
1502
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1503
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1504
|
+
with(basic_auth: ['john', 'doe']).
|
|
1505
|
+
to_return(status: 404, body: "", headers: {})
|
|
1506
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1507
|
+
with(basic_auth: ['john', 'doe'],
|
|
1508
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1509
|
+
to_return(status: 200, body: "", headers: {})
|
|
1510
|
+
# put the alias for the index
|
|
1511
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1512
|
+
with(basic_auth: ['john', 'doe']).
|
|
1513
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1514
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1515
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1516
|
+
to_return(status: 200, body: "", headers: {})
|
|
1517
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1518
|
+
with(basic_auth: ['john', 'doe']).
|
|
1519
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1520
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1521
|
+
with(basic_auth: ['john', 'doe']).
|
|
1522
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1523
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1524
|
+
with(basic_auth: ['john', 'doe'],
|
|
1525
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1526
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1527
|
+
|
|
1528
|
+
driver(config)
|
|
1529
|
+
|
|
1530
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1531
|
+
driver.run(default_tag: 'fluentd-policy') do
|
|
1532
|
+
driver.feed(sample_record)
|
|
1533
|
+
end
|
|
1534
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1535
|
+
|
|
1536
|
+
assert_requested(elastic_request)
|
|
1537
|
+
end
|
|
1538
|
+
|
|
1539
|
+
def test_tag_placeholder_with_multiple_policies
|
|
1540
|
+
cwd = File.dirname(__FILE__)
|
|
1541
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1542
|
+
|
|
1543
|
+
config = %{
|
|
1544
|
+
host logs.google.com
|
|
1545
|
+
port 777
|
|
1546
|
+
scheme https
|
|
1547
|
+
path /es/
|
|
1548
|
+
user john
|
|
1549
|
+
password doe
|
|
1550
|
+
template_name logstash
|
|
1551
|
+
template_file #{template_file}
|
|
1552
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1553
|
+
ilm_policy_id ${tag}
|
|
1554
|
+
enable_ilm true
|
|
1555
|
+
index_name logstash
|
|
1556
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
# connection start
|
|
1560
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1561
|
+
with(basic_auth: ['john', 'doe']).
|
|
1562
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1563
|
+
# check if template exists
|
|
1564
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1565
|
+
with(basic_auth: ['john', 'doe']).
|
|
1566
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1567
|
+
# creation
|
|
1568
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1569
|
+
with(basic_auth: ['john', 'doe']).
|
|
1570
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1571
|
+
# check if alias exists
|
|
1572
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1573
|
+
with(basic_auth: ['john', 'doe']).
|
|
1574
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1575
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1576
|
+
with(basic_auth: ['john', 'doe']).
|
|
1577
|
+
to_return(status: 404, body: "", headers: {})
|
|
1578
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1579
|
+
with(basic_auth: ['john', 'doe'],
|
|
1580
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1581
|
+
to_return(status: 200, body: "", headers: {})
|
|
1582
|
+
# put the alias for the index
|
|
1583
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1584
|
+
with(basic_auth: ['john', 'doe']).
|
|
1585
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1586
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1587
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1588
|
+
to_return(status: 200, body: "", headers: {})
|
|
1589
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1590
|
+
with(basic_auth: ['john', 'doe']).
|
|
1591
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1592
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1593
|
+
with(basic_auth: ['john', 'doe']).
|
|
1594
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1595
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1596
|
+
with(basic_auth: ['john', 'doe'],
|
|
1597
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
|
1598
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1599
|
+
|
|
1600
|
+
driver(config)
|
|
1601
|
+
|
|
1602
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1603
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
|
1604
|
+
driver.feed(sample_record)
|
|
1605
|
+
end
|
|
1606
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1607
|
+
|
|
1608
|
+
assert_requested(elastic_request)
|
|
1609
|
+
end
|
|
1610
|
+
end
|
|
1611
|
+
|
|
1302
1612
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
|
1303
1613
|
cwd = File.dirname(__FILE__)
|
|
1304
1614
|
template_file = File.join(cwd, 'test_template.json')
|
|
@@ -1371,6 +1681,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1371
1681
|
|
|
1372
1682
|
assert_requested(elastic_request)
|
|
1373
1683
|
end
|
|
1684
|
+
|
|
1685
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
|
|
1686
|
+
cwd = File.dirname(__FILE__)
|
|
1687
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1688
|
+
|
|
1689
|
+
config = Fluent::Config::Element.new(
|
|
1690
|
+
'ROOT', '', {
|
|
1691
|
+
'@type' => 'elasticsearch',
|
|
1692
|
+
'host' => 'logs.google.com',
|
|
1693
|
+
'port' => 777,
|
|
1694
|
+
'scheme' => "https",
|
|
1695
|
+
'path' => "/es/",
|
|
1696
|
+
'user' => 'john',
|
|
1697
|
+
'password' => 'doe',
|
|
1698
|
+
'template_name' => 'logstash',
|
|
1699
|
+
'template_file' => "#{template_file}",
|
|
1700
|
+
'index_date_pattern' => 'now/w{xxxx.ww}',
|
|
1701
|
+
'index_name' => "${taskDef}-%Y.%m",
|
|
1702
|
+
'enable_ilm' => true,
|
|
1703
|
+
}, [
|
|
1704
|
+
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
|
1705
|
+
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
|
1706
|
+
'timekey' => 3600,
|
|
1707
|
+
}, [])
|
|
1708
|
+
]
|
|
1709
|
+
)
|
|
1710
|
+
|
|
1711
|
+
task_def_value = "task_definition"
|
|
1712
|
+
date_str = Time.now.strftime("%Y.%m")
|
|
1713
|
+
# connection start
|
|
1714
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1715
|
+
with(basic_auth: ['john', 'doe']).
|
|
1716
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1717
|
+
# check if template exists
|
|
1718
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1719
|
+
with(basic_auth: ['john', 'doe']).
|
|
1720
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1721
|
+
# creation
|
|
1722
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1723
|
+
with(basic_auth: ['john', 'doe'],
|
|
1724
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
|
1725
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1726
|
+
# check if alias exists
|
|
1727
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
|
1728
|
+
with(basic_auth: ['john', 'doe']).
|
|
1729
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1730
|
+
# put the alias for the index
|
|
1731
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1732
|
+
with(basic_auth: ['john', 'doe']).
|
|
1733
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1734
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
|
|
1735
|
+
with(basic_auth: ['john', 'doe'],
|
|
1736
|
+
:body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1737
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1738
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1739
|
+
with(basic_auth: ['john', 'doe']).
|
|
1740
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1741
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1742
|
+
with(basic_auth: ['john', 'doe']).
|
|
1743
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1744
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1745
|
+
with(basic_auth: ['john', 'doe'],
|
|
1746
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1747
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1748
|
+
|
|
1749
|
+
driver(config)
|
|
1750
|
+
|
|
1751
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1752
|
+
driver.run(default_tag: 'test') do
|
|
1753
|
+
driver.feed(sample_record.merge("taskDef" => task_def_value))
|
|
1754
|
+
end
|
|
1755
|
+
assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
|
|
1756
|
+
|
|
1757
|
+
assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
|
|
1758
|
+
|
|
1759
|
+
assert_requested(elastic_request)
|
|
1760
|
+
end
|
|
1374
1761
|
end
|
|
1375
1762
|
|
|
1376
1763
|
def test_custom_template_create
|
|
@@ -1627,11 +2014,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1627
2014
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1628
2015
|
# check if alias exists
|
|
1629
2016
|
timestr = Time.now.strftime("%Y.%m.%d")
|
|
1630
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
|
2017
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
|
|
1631
2018
|
with(basic_auth: ['john', 'doe']).
|
|
1632
2019
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1633
2020
|
# put the alias for the index
|
|
1634
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
|
2021
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-myapp-#{timestr}").
|
|
1635
2022
|
with(basic_auth: ['john', 'doe']).
|
|
1636
2023
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1637
2024
|
|
|
@@ -2911,6 +3298,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2911
3298
|
|
|
2912
3299
|
data("border" => {"es_version" => 6, "_type" => "mytype"},
|
|
2913
3300
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3301
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2914
3302
|
)
|
|
2915
3303
|
def test_writes_to_speficied_type(data)
|
|
2916
3304
|
driver('', data["es_version"]).configure("type_name mytype\n")
|
|
@@ -2923,6 +3311,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2923
3311
|
|
|
2924
3312
|
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
2925
3313
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3314
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2926
3315
|
)
|
|
2927
3316
|
def test_writes_to_speficied_type_with_placeholders(data)
|
|
2928
3317
|
driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
|
|
@@ -2933,6 +3322,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2933
3322
|
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
2934
3323
|
end
|
|
2935
3324
|
|
|
3325
|
+
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
3326
|
+
"fixed_behavior"=> {"es_version" => 7, "_type" => nil},
|
|
3327
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
3328
|
+
)
|
|
3329
|
+
def test_writes_to_speficied_type_with_suppress_type_name(data)
|
|
3330
|
+
driver('', data["es_version"])
|
|
3331
|
+
.configure("type_name mytype.${tag}\nsuppress_type_name true")
|
|
3332
|
+
stub_elastic
|
|
3333
|
+
driver.run(default_tag: 'test') do
|
|
3334
|
+
driver.feed(sample_record)
|
|
3335
|
+
end
|
|
3336
|
+
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
3337
|
+
end
|
|
3338
|
+
|
|
2936
3339
|
data("old" => {"es_version" => 2, "_type" => "local-override"},
|
|
2937
3340
|
"old_behavior" => {"es_version" => 5, "_type" => "local-override"},
|
|
2938
3341
|
"border" => {"es_version" => 6, "_type" => "fluentd"},
|