fluent-plugin-elasticsearch 2.11.5 → 2.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +3 -0
- data/README.md +23 -1
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_index_template.rb +42 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +7 -1
- data/test/plugin/test_alias_template.json +9 -0
- data/test/plugin/test_out_elasticsearch.rb +82 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1a97d5261421d51521687b644f6f6b02555ad0f1fd592b24a40f876a8702ba4e
|
4
|
+
data.tar.gz: b31b34473d410112dd2d1d6e05ac51d277953bb2d33e411eaa6621731cbc5052
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d9708a487c86d1cdb367789fccf3ae0ff9a7900710e803e4ca2bcc22f5216334775ae806a86e9b08c09f473bda7c15fa3ba3a5df71ccd95a2de6f3a19fc6d3c5
|
7
|
+
data.tar.gz: 541256741abbb757a29ba855a19f1d880240f1b6a05d3583017b3f23b600b1bd987f8f8734f627e34868b4ba8d7083c0534d8c10878c5f0ea6216a47722416d1
|
data/History.md
CHANGED
data/README.md
CHANGED
@@ -36,6 +36,8 @@ Current maintainers: @cosmo0920
|
|
36
36
|
+ [template_name](#template_name)
|
37
37
|
+ [template_file](#template_file)
|
38
38
|
+ [template_overwrite](#template_overwrite)
|
39
|
+
+ [customize_template](#customize_template)
|
40
|
+
+ [index_prefix](#index_prefix)
|
39
41
|
+ [templates](#templates)
|
40
42
|
+ [max_retry_putting_template](#max_retry_putting_template)
|
41
43
|
+ [request_timeout](#request_timeout)
|
@@ -127,12 +129,13 @@ hosts host1:port1,host2:port2,host3:port3
|
|
127
129
|
You can specify multiple Elasticsearch hosts with separator ",".
|
128
130
|
|
129
131
|
If you specify multiple hosts, this plugin will load balance updates to Elasticsearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
|
132
|
+
**Note:** If you will use scheme https, do not include "https://" in your hosts ie. host "https://domain", this will cause ES cluster to be unreachable and you will recieve an error "Can not reach Elasticsearch cluster"
|
130
133
|
|
131
134
|
**Note:** Up until v2.8.5, it was allowed to embed the username/password in the URL. However, this syntax is deprecated as of v2.8.6 because it was found to cause serious connection problems (See #394). Please migrate your settings to use the `user` and `password` field (described below) instead.
|
132
135
|
|
133
136
|
### user, password, path, scheme, ssl_verify
|
134
137
|
|
135
|
-
If you specify this option,
|
138
|
+
If you specify this option, port options are ignored.
|
136
139
|
|
137
140
|
```
|
138
141
|
user demo
|
@@ -331,6 +334,25 @@ templates { "template_name_1": "path_to_template_1_file", "template_name_2": "pa
|
|
331
334
|
|
332
335
|
If `template_file` and `template_name` are set, then this parameter will be ignored.
|
333
336
|
|
337
|
+
### customize_template
|
338
|
+
|
339
|
+
Specify the string and its value to be replaced in form of hash. Can contain multiple templates.
|
340
|
+
|
341
|
+
```
|
342
|
+
customize_template {"string_1": "subs_value_1", "string_2": "subs_value_2"}
|
343
|
+
```
|
344
|
+
|
345
|
+
If `template_file` and `template_name` are set, then this parameter will be in effect otherwise ignored.
|
346
|
+
|
347
|
+
### index_prefix
|
348
|
+
|
349
|
+
```
|
350
|
+
index_prefix mylogs # defaults to "logstash"
|
351
|
+
```
|
352
|
+
|
353
|
+
If `customize_template` is set, then this parameter will be in effect otherwise ignored.
|
354
|
+
|
355
|
+
|
334
356
|
### template_overwrite
|
335
357
|
|
336
358
|
Always update the template, even if it already exists.
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '2.11.
|
6
|
+
s.version = '2.11.6'
|
7
7
|
s.authors = ['diogo', 'pitr']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -8,6 +8,17 @@ module Fluent::ElasticsearchIndexTemplate
|
|
8
8
|
JSON.parse(file_contents)
|
9
9
|
end
|
10
10
|
|
11
|
+
def get_custom_template(template_file, customize_template)
|
12
|
+
if !File.exists?(template_file)
|
13
|
+
raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
|
14
|
+
end
|
15
|
+
file_contents = IO.read(template_file).gsub(/\n/,'')
|
16
|
+
customize_template.each do |key, value|
|
17
|
+
file_contents = file_contents.gsub(key,value.downcase)
|
18
|
+
end
|
19
|
+
JSON.parse(file_contents)
|
20
|
+
end
|
21
|
+
|
11
22
|
def template_exists?(name)
|
12
23
|
client.indices.get_template(:name => name)
|
13
24
|
return true
|
@@ -37,6 +48,12 @@ module Fluent::ElasticsearchIndexTemplate
|
|
37
48
|
client.indices.put_template(:name => name, :body => template)
|
38
49
|
end
|
39
50
|
|
51
|
+
def indexcreation(index_name)
|
52
|
+
client.indices.create(:index => index_name)
|
53
|
+
rescue Elasticsearch::Transport::Transport::Error => e
|
54
|
+
log.error("Error while index creation - #{index_name}: #{e.inspect}")
|
55
|
+
end
|
56
|
+
|
40
57
|
def template_install(name, template_file, overwrite)
|
41
58
|
if overwrite
|
42
59
|
template_put(name, get_template(template_file))
|
@@ -51,6 +68,31 @@ module Fluent::ElasticsearchIndexTemplate
|
|
51
68
|
end
|
52
69
|
end
|
53
70
|
|
71
|
+
def template_custom_install(name, template_file, overwrite, customize_template, index_prefix)
|
72
|
+
template_custom_name=name.downcase+'_alias_template'
|
73
|
+
alias_name=name.downcase+'-current'
|
74
|
+
if overwrite
|
75
|
+
template_put(template_custom_name, get_custom_template(template_file, customize_template))
|
76
|
+
log.info("Template '#{template_custom_name}' overwritten with #{template_file}.")
|
77
|
+
return
|
78
|
+
end
|
79
|
+
if !template_exists?(template_custom_name)
|
80
|
+
template_put(template_custom_name, get_custom_template(template_file, customize_template))
|
81
|
+
log.info("Template configured, but no template installed. Installed '#{template_custom_name}' from #{template_file}.")
|
82
|
+
else
|
83
|
+
log.info("Template configured and already installed.")
|
84
|
+
end
|
85
|
+
|
86
|
+
if !client.indices.exists_alias(:name => alias_name)
|
87
|
+
index_name='<'+index_prefix.downcase+'-'+name.downcase+'-{now/d}-000001>'
|
88
|
+
indexcreation(index_name)
|
89
|
+
client.indices.put_alias(:index => index_name, :name => alias_name)
|
90
|
+
log.info("The alias '#{alias_name}' is created for the index '#{index_name}'")
|
91
|
+
else
|
92
|
+
log.info("The alias '#{alias_name}' is already present")
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
54
96
|
def templates_hash_install(templates, overwrite)
|
55
97
|
templates.each do |key, value|
|
56
98
|
template_install(key, value, overwrite)
|
@@ -93,6 +93,8 @@ EOC
|
|
93
93
|
config_param :template_name, :string, :default => nil
|
94
94
|
config_param :template_file, :string, :default => nil
|
95
95
|
config_param :template_overwrite, :bool, :default => false
|
96
|
+
config_param :customize_template, :hash, :default => nil
|
97
|
+
config_param :index_prefix, :string, :default => "logstash"
|
96
98
|
config_param :templates, :hash, :default => nil
|
97
99
|
config_param :max_retry_putting_template, :integer, :default => 10
|
98
100
|
config_param :include_tag_key, :bool, :default => false
|
@@ -153,7 +155,11 @@ EOC
|
|
153
155
|
raise Fluent::ConfigError, "'max_retry_putting_template' must be positive number." if @max_retry_putting_template < 0
|
154
156
|
if @template_name && @template_file
|
155
157
|
retry_install(@max_retry_putting_template) do
|
156
|
-
|
158
|
+
if @customize_template
|
159
|
+
template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @index_prefix)
|
160
|
+
else
|
161
|
+
template_install(@template_name, @template_file, @template_overwrite)
|
162
|
+
end
|
157
163
|
end
|
158
164
|
elsif @templates
|
159
165
|
retry_install(@max_retry_putting_template) do
|
@@ -340,6 +340,47 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
340
340
|
assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
|
341
341
|
end
|
342
342
|
|
343
|
+
def test_custom_template_create
|
344
|
+
cwd = File.dirname(__FILE__)
|
345
|
+
template_file = File.join(cwd, 'test_alias_template.json')
|
346
|
+
|
347
|
+
config = %{
|
348
|
+
host logs.google.com
|
349
|
+
port 777
|
350
|
+
scheme https
|
351
|
+
path /es/
|
352
|
+
user john
|
353
|
+
password doe
|
354
|
+
template_name myapp
|
355
|
+
template_file #{template_file}
|
356
|
+
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
357
|
+
index_prefix mylogs
|
358
|
+
}
|
359
|
+
|
360
|
+
# connection start
|
361
|
+
stub_request(:head, "https://john:doe@logs.google.com:777/es//").
|
362
|
+
to_return(:status => 200, :body => "", :headers => {})
|
363
|
+
# check if template exists
|
364
|
+
stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
|
365
|
+
to_return(:status => 404, :body => "", :headers => {})
|
366
|
+
# creation
|
367
|
+
stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
|
368
|
+
to_return(:status => 200, :body => "", :headers => {})
|
369
|
+
# creation of index which can rollover
|
370
|
+
stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
|
371
|
+
to_return(:status => 200, :body => "", :headers => {})
|
372
|
+
# check if alias exists
|
373
|
+
stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp-current").
|
374
|
+
to_return(:status => 404, :body => "", :headers => {})
|
375
|
+
# put the alias for the index
|
376
|
+
stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp-current").
|
377
|
+
to_return(:status => 200, :body => "", :headers => {})
|
378
|
+
|
379
|
+
driver(config)
|
380
|
+
|
381
|
+
assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
|
382
|
+
end
|
383
|
+
|
343
384
|
def test_template_overwrite
|
344
385
|
cwd = File.dirname(__FILE__)
|
345
386
|
template_file = File.join(cwd, 'test_template.json')
|
@@ -371,6 +412,47 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
371
412
|
assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
|
372
413
|
end
|
373
414
|
|
415
|
+
def test_custom_template_overwrite
|
416
|
+
cwd = File.dirname(__FILE__)
|
417
|
+
template_file = File.join(cwd, 'test_template.json')
|
418
|
+
|
419
|
+
config = %{
|
420
|
+
host logs.google.com
|
421
|
+
port 777
|
422
|
+
scheme https
|
423
|
+
path /es/
|
424
|
+
user john
|
425
|
+
password doe
|
426
|
+
template_name myapp
|
427
|
+
template_file #{template_file}
|
428
|
+
template_overwrite true
|
429
|
+
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
430
|
+
index_prefix mylogs
|
431
|
+
}
|
432
|
+
|
433
|
+
# connection start
|
434
|
+
stub_request(:head, "https://john:doe@logs.google.com:777/es//").
|
435
|
+
to_return(:status => 200, :body => "", :headers => {})
|
436
|
+
# check if template exists
|
437
|
+
stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
|
438
|
+
to_return(:status => 200, :body => "", :headers => {})
|
439
|
+
# creation
|
440
|
+
stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
|
441
|
+
to_return(:status => 200, :body => "", :headers => {})
|
442
|
+
# creation of index which can rollover
|
443
|
+
stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
|
444
|
+
to_return(:status => 200, :body => "", :headers => {})
|
445
|
+
# check if alias exists
|
446
|
+
stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp-current").
|
447
|
+
to_return(:status => 404, :body => "", :headers => {})
|
448
|
+
# put the alias for the index
|
449
|
+
stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp-current").
|
450
|
+
to_return(:status => 200, :body => "", :headers => {})
|
451
|
+
|
452
|
+
driver(config)
|
453
|
+
|
454
|
+
assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template", times: 1)
|
455
|
+
end
|
374
456
|
|
375
457
|
def test_template_create_invalid_filename
|
376
458
|
config = %{
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.11.
|
4
|
+
version: 2.11.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-08-
|
12
|
+
date: 2018-08-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -152,6 +152,7 @@ files:
|
|
152
152
|
- lib/fluent/plugin/out_elasticsearch.rb
|
153
153
|
- lib/fluent/plugin/out_elasticsearch_dynamic.rb
|
154
154
|
- test/helper.rb
|
155
|
+
- test/plugin/test_alias_template.json
|
155
156
|
- test/plugin/test_elasticsearch_error_handler.rb
|
156
157
|
- test/plugin/test_filter_elasticsearch_genid.rb
|
157
158
|
- test/plugin/test_out_elasticsearch.rb
|
@@ -183,6 +184,7 @@ specification_version: 4
|
|
183
184
|
summary: Elasticsearch output plugin for Fluent event collector
|
184
185
|
test_files:
|
185
186
|
- test/helper.rb
|
187
|
+
- test/plugin/test_alias_template.json
|
186
188
|
- test/plugin/test_elasticsearch_error_handler.rb
|
187
189
|
- test/plugin/test_filter_elasticsearch_genid.rb
|
188
190
|
- test/plugin/test_out_elasticsearch.rb
|