fluent-plugin-elasticsearch 4.0.4 → 4.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +23 -2
- data/Gemfile +1 -0
- data/History.md +7 -0
- data/README.md +15 -1
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
- data/gemfiles/Gemfile.ilm +1 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +7 -7
- data/test/plugin/test_oj_serializer.rb +19 -0
- data/test/plugin/test_out_elasticsearch.rb +86 -20
- data/test/plugin/test_out_elasticsearch_dynamic.rb +10 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 81cf6831267be2a7250ab5365cf783c0680e5065ff3e3df068707ab3528bdbfb
|
4
|
+
data.tar.gz: ea99a64f901f9daa9cbf1ac43bc51064048765872b9c1d29a3029dd9b9f56f24
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6c750270cf3051fe974ebaad5063c82cd8eb9be3afc89326ce99cd5929890f2330b2397ef2fa683f1a060f035db2266862666e09282b60ef8102cc497d2a888b
|
7
|
+
data.tar.gz: bee1bc70e85c8c475d943abb922531c5e668bcd7457692d006cff1fb59a669845ce8ad4a1e6a45cdca58e4a0bc63626638eeef50714bf67d6ff1c7f1a1c3c5fb
|
data/.travis.yml
CHANGED
@@ -1,17 +1,39 @@
|
|
1
1
|
language: ruby
|
2
2
|
|
3
|
-
|
3
|
+
jobs:
|
4
4
|
include:
|
5
5
|
- rvm: 2.4.6
|
6
6
|
gemfile: Gemfile
|
7
|
+
os: linux
|
8
|
+
arch: amd64
|
7
9
|
- rvm: 2.5.5
|
8
10
|
gemfile: Gemfile
|
11
|
+
os: linux
|
12
|
+
arch: amd64
|
9
13
|
- rvm: 2.6.3
|
10
14
|
gemfile: gemfiles/Gemfile.ilm
|
15
|
+
os: linux
|
16
|
+
arch: amd64
|
17
|
+
- rvm: 2.6.3
|
18
|
+
gemfile: gemfiles/Gemfile.elasticsearch.v6
|
19
|
+
os: linux
|
20
|
+
arch: amd64
|
21
|
+
- rvm: 2.6.3
|
22
|
+
gemfile: Gemfile
|
23
|
+
os: linux
|
24
|
+
arch: amd64
|
25
|
+
- rvm: 2.6.3
|
26
|
+
gemfile: Gemfile
|
27
|
+
os: linux
|
28
|
+
arch: arm64
|
11
29
|
- rvm: 2.6.3
|
12
30
|
gemfile: Gemfile
|
31
|
+
os: osx
|
32
|
+
osx_image: xcode11.3
|
13
33
|
- rvm: 2.7.0
|
14
34
|
gemfile: Gemfile
|
35
|
+
os: linux
|
36
|
+
arch: amd64
|
15
37
|
|
16
38
|
gemfile:
|
17
39
|
- Gemfile
|
@@ -20,4 +42,3 @@ before_install:
|
|
20
42
|
- gem update --system=2.7.8
|
21
43
|
|
22
44
|
script: bundle exec rake test
|
23
|
-
sudo: false
|
data/Gemfile
CHANGED
data/History.md
CHANGED
@@ -2,6 +2,13 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 4.0.5
|
6
|
+
- add logstash_dateformat as placeholder (#718)
|
7
|
+
- Tweak travis.yml for suppressing validator warnings and add CI for Linux Arm64 architecture and macOS 10.14 (#724)
|
8
|
+
- Elasticsearch ruby v7.5 (#723)
|
9
|
+
- Add Oj serializer testcases for all job (#722)
|
10
|
+
- Update documentation for ILM (#721)
|
11
|
+
|
5
12
|
### 4.0.4
|
6
13
|
- Provide clearing caches timer (#719)
|
7
14
|
|
data/README.md
CHANGED
@@ -1639,17 +1639,31 @@ Main ILM feature parameters are:
|
|
1639
1639
|
* `ilm_policy_id`
|
1640
1640
|
* `ilm_policy`
|
1641
1641
|
|
1642
|
+
* Advanced usage parameters
|
1643
|
+
* `application_name`
|
1644
|
+
* `index_separator`
|
1645
|
+
|
1642
1646
|
They are not all mandatory parameters but they are used for ILM feature in effect.
|
1643
1647
|
|
1644
1648
|
ILM target index alias is created with `index_name` or an index which is calculated from `logstash_prefix`.
|
1645
1649
|
|
1646
1650
|
From Elasticsearch plugin v4.0.0, ILM target index will be calculated from `index_name` (normal mode) or `logstash_prefix` (using with `logstash_format`as true).
|
1647
1651
|
|
1648
|
-
When using `
|
1652
|
+
When using `deflector_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
|
1649
1653
|
|
1650
1654
|
And also, ILM feature users should specify their Elasticsearch template for ILM enabled indices.
|
1651
1655
|
Because ILM settings are injected into their Elasticsearch templates.
|
1652
1656
|
|
1657
|
+
`application_name` and `index_separator` also affect alias index names.
|
1658
|
+
|
1659
|
+
But this parameter is prepared for advanced usage.
|
1660
|
+
|
1661
|
+
It usually should be used with default value which is `default`.
|
1662
|
+
|
1663
|
+
Then, ILM parameters are used in alias index like as:
|
1664
|
+
|
1665
|
+
`<index_name/logstash_prefix><index_separator><application_name>-000001`.
|
1666
|
+
|
1653
1667
|
#### Example ILM settings
|
1654
1668
|
|
1655
1669
|
```aconf
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.0.
|
6
|
+
s.version = '4.0.5'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -0,0 +1,12 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
|
4
|
+
gemspec :path => "../"
|
5
|
+
|
6
|
+
gem 'simplecov', require: false
|
7
|
+
gem 'coveralls', require: false
|
8
|
+
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
|
+
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
|
+
gem "elasticsearch", "~> 6.8.1"
|
11
|
+
gem "elasticsearch-xpack"
|
12
|
+
gem "oj"
|
data/gemfiles/Gemfile.ilm
CHANGED
@@ -358,9 +358,7 @@ EOC
|
|
358
358
|
end
|
359
359
|
end
|
360
360
|
|
361
|
-
|
362
|
-
|
363
|
-
if (version_arr[0].to_i < 7) || (version_arr[0].to_i == 7 && version_arr[1].to_i < 2)
|
361
|
+
if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
364
362
|
if compression
|
365
363
|
raise Fluent::ConfigError, <<-EOC
|
366
364
|
Cannot use compression with elasticsearch-transport plugin version < 7.2.0
|
@@ -669,6 +667,7 @@ EOC
|
|
669
667
|
|
670
668
|
def expand_placeholders(chunk)
|
671
669
|
logstash_prefix = extract_placeholders(@logstash_prefix, chunk)
|
670
|
+
logstash_dateformat = extract_placeholders(@logstash_dateformat, chunk)
|
672
671
|
index_name = extract_placeholders(@index_name, chunk)
|
673
672
|
if @type_name
|
674
673
|
type_name = extract_placeholders(@type_name, chunk)
|
@@ -700,7 +699,7 @@ EOC
|
|
700
699
|
else
|
701
700
|
pipeline = nil
|
702
701
|
end
|
703
|
-
return logstash_prefix, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
|
702
|
+
return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
|
704
703
|
end
|
705
704
|
|
706
705
|
def multi_workers_ready?
|
@@ -774,7 +773,7 @@ EOC
|
|
774
773
|
end
|
775
774
|
|
776
775
|
def process_message(tag, meta, header, time, record, extracted_values)
|
777
|
-
logstash_prefix, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
|
776
|
+
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
|
778
777
|
|
779
778
|
if @flatten_hashes
|
780
779
|
record = flatten_record(record)
|
@@ -800,7 +799,7 @@ EOC
|
|
800
799
|
target_index = target_index_parent.delete(target_index_child_key)
|
801
800
|
elsif @logstash_format
|
802
801
|
dt = dt.new_offset(0) if @utc_index
|
803
|
-
target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(
|
802
|
+
target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(logstash_dateformat)}"
|
804
803
|
else
|
805
804
|
target_index = index_name
|
806
805
|
end
|
@@ -880,6 +879,7 @@ EOC
|
|
880
879
|
placeholder?(:template_name, @template_name.to_s) ||
|
881
880
|
@customize_template&.values&.any? { |value| placeholder?(:customize_template, value.to_s) } ||
|
882
881
|
placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
|
882
|
+
placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
|
883
883
|
placeholder?(:deflector_alias, @deflector_alias.to_s) ||
|
884
884
|
placeholder?(:application_name, @application_name.to_s)
|
885
885
|
log.debug("Need substitution: #{need_substitution}")
|
@@ -914,7 +914,7 @@ EOC
|
|
914
914
|
# send_bulk given a specific bulk request, the original tag,
|
915
915
|
# chunk, and bulk_message_count
|
916
916
|
def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
|
917
|
-
logstash_prefix, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
|
917
|
+
logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
|
918
918
|
if deflector_alias
|
919
919
|
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
|
920
920
|
else
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'elasticsearch'
|
3
|
+
|
4
|
+
class OjSerializerTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
begin
|
7
|
+
require 'fluent/plugin/oj_serializer'
|
8
|
+
rescue LoadError
|
9
|
+
omit "OjSerializer testcase needs oj gem."
|
10
|
+
end
|
11
|
+
@serializer = Fluent::Plugin::Serializer::Oj.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_serializer
|
15
|
+
data = {"message" => "Hi"}
|
16
|
+
assert_equal data.to_json, @serializer.dump(data)
|
17
|
+
assert_equal data, @serializer.load(data.to_json)
|
18
|
+
end
|
19
|
+
end
|
@@ -201,6 +201,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
201
201
|
assert_compare(exp_matches, operator, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
|
202
202
|
end
|
203
203
|
|
204
|
+
def alias_endpoint
|
205
|
+
if Gem::Version.create(::Elasticsearch::VERSION) >= Gem::Version.create("7.5.0-pre")
|
206
|
+
"_aliases"
|
207
|
+
else
|
208
|
+
"_alias"
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
204
212
|
def test_configure
|
205
213
|
config = %{
|
206
214
|
host logs.google.com
|
@@ -246,6 +254,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
246
254
|
end
|
247
255
|
|
248
256
|
test 'configure compression' do
|
257
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
258
|
+
|
249
259
|
config = %{
|
250
260
|
compression_level best_compression
|
251
261
|
}
|
@@ -255,6 +265,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
255
265
|
end
|
256
266
|
|
257
267
|
test 'check compression strategy' do
|
268
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
269
|
+
|
258
270
|
config = %{
|
259
271
|
compression_level best_speed
|
260
272
|
}
|
@@ -264,6 +276,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
264
276
|
end
|
265
277
|
|
266
278
|
test 'check content-encoding header with compression' do
|
279
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
280
|
+
|
267
281
|
config = %{
|
268
282
|
compression_level best_compression
|
269
283
|
}
|
@@ -273,6 +287,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
273
287
|
end
|
274
288
|
|
275
289
|
test 'check compression option is passed to transport' do
|
290
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
291
|
+
|
276
292
|
config = %{
|
277
293
|
compression_level best_compression
|
278
294
|
}
|
@@ -338,7 +354,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
338
354
|
to_return(status: 200, body: "", headers: {})
|
339
355
|
stub_request(:head, "http://localhost:9200/_alias/fluentd").
|
340
356
|
to_return(status: 404, body: "", headers: {})
|
341
|
-
stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/
|
357
|
+
stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/#{alias_endpoint}/fluentd").
|
342
358
|
with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
|
343
359
|
to_return(status: 200, body: "", headers: {})
|
344
360
|
stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
|
@@ -372,7 +388,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
372
388
|
to_return(status: 200, body: "", headers: {})
|
373
389
|
stub_request(:head, "http://localhost:9200/_alias/fluentd").
|
374
390
|
to_return(status: 404, body: "", headers: {})
|
375
|
-
stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/
|
391
|
+
stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E/#{alias_endpoint}/fluentd").
|
376
392
|
with(body: "{\"aliases\":{\"fluentd\":{\"is_write_index\":true}}}").
|
377
393
|
to_return(status: 200, body: "", headers: {})
|
378
394
|
stub_request(:put, "http://localhost:9200/%3Cfluentd-default-%7Bnow%2Fd%7D-000001%3E").
|
@@ -791,7 +807,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
791
807
|
stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E").
|
792
808
|
with(basic_auth: ['john', 'doe']).
|
793
809
|
to_return(status: 200, body: "", headers: {})
|
794
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/
|
810
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/#{alias_endpoint}/myapp_deflector-test.template").
|
795
811
|
with(basic_auth: ['john', 'doe'],
|
796
812
|
body: "{\"aliases\":{\"myapp_deflector-test.template\":{\"is_write_index\":true}}}").
|
797
813
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -849,7 +865,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
849
865
|
stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E").
|
850
866
|
with(basic_auth: ['john', 'doe']).
|
851
867
|
to_return(status: 200, body: "", headers: {})
|
852
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/
|
868
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cfluentd-test.template-default-000001%3E/#{alias_endpoint}/myapp_deflector-test.template").
|
853
869
|
with(basic_auth: ['john', 'doe'],
|
854
870
|
body: "{\"aliases\":{\"myapp_deflector-test.template\":{\"is_write_index\":true}}}").
|
855
871
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -928,7 +944,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
928
944
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
929
945
|
with(basic_auth: ['john', 'doe']).
|
930
946
|
to_return(:status => 200, :body => "", :headers => {})
|
931
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
947
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
932
948
|
with(basic_auth: ['john', 'doe'],
|
933
949
|
:body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
934
950
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -995,7 +1011,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
995
1011
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
996
1012
|
with(basic_auth: ['john', 'doe']).
|
997
1013
|
to_return(:status => 200, :body => "", :headers => {})
|
998
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1014
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
999
1015
|
with(basic_auth: ['john', 'doe'],
|
1000
1016
|
:body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1001
1017
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1061,7 +1077,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1061
1077
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1062
1078
|
with(basic_auth: ['john', 'doe']).
|
1063
1079
|
to_return(:status => 200, :body => "", :headers => {})
|
1064
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1080
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
1065
1081
|
with(basic_auth: ['john', 'doe'],
|
1066
1082
|
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
1067
1083
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1126,7 +1142,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1126
1142
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
1127
1143
|
with(basic_auth: ['john', 'doe']).
|
1128
1144
|
to_return(:status => 200, :body => "", :headers => {})
|
1129
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/
|
1145
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/#{alias_endpoint}/logstash").
|
1130
1146
|
with(basic_auth: ['john', 'doe'],
|
1131
1147
|
:body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1132
1148
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1193,7 +1209,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1193
1209
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1194
1210
|
with(basic_auth: ['john', 'doe']).
|
1195
1211
|
to_return(:status => 200, :body => "", :headers => {})
|
1196
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1212
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1197
1213
|
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1198
1214
|
to_return(status: 200, body: "", headers: {})
|
1199
1215
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
@@ -1257,7 +1273,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1257
1273
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1258
1274
|
with(basic_auth: ['john', 'doe']).
|
1259
1275
|
to_return(:status => 200, :body => "", :headers => {})
|
1260
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1276
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-test").
|
1261
1277
|
with(basic_auth: ['john', 'doe'],
|
1262
1278
|
:body => "{\"aliases\":{\"logstash-test\":{\"is_write_index\":true}}}").
|
1263
1279
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1441,7 +1457,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1441
1457
|
with(basic_auth: ['john', 'doe']).
|
1442
1458
|
to_return(:status => 404, :body => "", :headers => {})
|
1443
1459
|
# put the alias for the index
|
1444
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1460
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
1445
1461
|
with(basic_auth: ['john', 'doe']).
|
1446
1462
|
to_return(:status => 200, :body => "", :headers => {})
|
1447
1463
|
|
@@ -1450,7 +1466,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1450
1466
|
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
|
1451
1467
|
end
|
1452
1468
|
|
1453
|
-
|
1469
|
+
def test_custom_template_with_rollover_index_create_and_deflector_alias
|
1454
1470
|
cwd = File.dirname(__FILE__)
|
1455
1471
|
template_file = File.join(cwd, 'test_alias_template.json')
|
1456
1472
|
|
@@ -1492,7 +1508,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1492
1508
|
with(basic_auth: ['john', 'doe']).
|
1493
1509
|
to_return(:status => 404, :body => "", :headers => {})
|
1494
1510
|
# put the alias for the index
|
1495
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1511
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
1496
1512
|
with(basic_auth: ['john', 'doe']).
|
1497
1513
|
to_return(:status => 200, :body => "", :headers => {})
|
1498
1514
|
|
@@ -1544,7 +1560,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1544
1560
|
with(basic_auth: ['john', 'doe']).
|
1545
1561
|
to_return(:status => 404, :body => "", :headers => {})
|
1546
1562
|
# put the alias for the index
|
1547
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1563
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
1548
1564
|
with(basic_auth: ['john', 'doe']).
|
1549
1565
|
to_return(:status => 200, :body => "", :headers => {})
|
1550
1566
|
|
@@ -1615,7 +1631,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1615
1631
|
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
|
1616
1632
|
to_return(status: 200, body: "", headers: {})
|
1617
1633
|
# put the alias for the index
|
1618
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1634
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
1619
1635
|
with(basic_auth: ['john', 'doe'],
|
1620
1636
|
:body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
|
1621
1637
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1686,7 +1702,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1686
1702
|
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
|
1687
1703
|
to_return(status: 200, body: "", headers: {})
|
1688
1704
|
# put the alias for the index
|
1689
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1705
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
1690
1706
|
with(basic_auth: ['john', 'doe'],
|
1691
1707
|
:body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
|
1692
1708
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1756,7 +1772,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1756
1772
|
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
|
1757
1773
|
to_return(status: 200, body: "", headers: {})
|
1758
1774
|
# put the alias for the index
|
1759
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1775
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
1760
1776
|
with(basic_auth: ['john', 'doe'],
|
1761
1777
|
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
1762
1778
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1825,7 +1841,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1825
1841
|
body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
|
1826
1842
|
to_return(status: 200, body: "", headers: {})
|
1827
1843
|
# put the alias for the index
|
1828
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1844
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-custom-test").
|
1829
1845
|
with(basic_auth: ['john', 'doe'],
|
1830
1846
|
:body => "{\"aliases\":{\"mylogs-custom-test\":{\"is_write_index\":true}}}").
|
1831
1847
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1902,7 +1918,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1902
1918
|
with(basic_auth: ['john', 'doe']).
|
1903
1919
|
to_return(status: 200, body: "", headers: {})
|
1904
1920
|
# put the alias for the index
|
1905
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/
|
1921
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
1906
1922
|
with(basic_auth: ['john', 'doe'],
|
1907
1923
|
:body => "{\"aliases\":{\"mylogs\":{\"is_write_index\":true}}}").
|
1908
1924
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -2034,7 +2050,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
2034
2050
|
with(basic_auth: ['john', 'doe']).
|
2035
2051
|
to_return(:status => 404, :body => "", :headers => {})
|
2036
2052
|
# put the alias for the index
|
2037
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/
|
2053
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
2038
2054
|
with(basic_auth: ['john', 'doe']).
|
2039
2055
|
to_return(:status => 200, :body => "", :headers => {})
|
2040
2056
|
|
@@ -2548,6 +2564,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
2548
2564
|
|
2549
2565
|
|
2550
2566
|
def test_writes_to_default_index_with_compression
|
2567
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
2568
|
+
|
2551
2569
|
config = %[
|
2552
2570
|
compression_level default_compression
|
2553
2571
|
]
|
@@ -3121,6 +3139,54 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
3121
3139
|
end
|
3122
3140
|
end
|
3123
3141
|
|
3142
|
+
class LogStashDateformatPlaceholdersTest < self
|
3143
|
+
def test_writes_to_logstash_index_with_specified_prefix_and_dateformat_placeholder_pattern_1
|
3144
|
+
driver.configure(Fluent::Config::Element.new(
|
3145
|
+
'ROOT', '', {
|
3146
|
+
'@type' => 'elasticsearch',
|
3147
|
+
'logstash_format' => true,
|
3148
|
+
'logstash_dateformat' => '${indexformat}',
|
3149
|
+
'logstash_prefix' => 'myprefix',
|
3150
|
+
}, [
|
3151
|
+
Fluent::Config::Element.new('buffer', 'tag,time,indexformat', {
|
3152
|
+
'chunk_keys' => ['tag', 'time', 'indexformat'],
|
3153
|
+
'timekey' => 3600,
|
3154
|
+
}, [])
|
3155
|
+
]
|
3156
|
+
))
|
3157
|
+
time = Time.parse Date.today.iso8601
|
3158
|
+
logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
|
3159
|
+
stub_elastic
|
3160
|
+
driver.run(default_tag: 'test') do
|
3161
|
+
driver.feed(time.to_i, sample_record.merge('indexformat' => '%Y.%m.%d'))
|
3162
|
+
end
|
3163
|
+
assert_equal(logstash_index, index_cmds.first['index']['_index'])
|
3164
|
+
end
|
3165
|
+
|
3166
|
+
def test_writes_to_logstash_index_with_specified_prefix_and_dateformat_placeholder_pattern_2
|
3167
|
+
driver.configure(Fluent::Config::Element.new(
|
3168
|
+
'ROOT', '', {
|
3169
|
+
'@type' => 'elasticsearch',
|
3170
|
+
'logstash_format' => true,
|
3171
|
+
'logstash_dateformat' => '${indexformat}',
|
3172
|
+
'logstash_prefix' => 'myprefix',
|
3173
|
+
}, [
|
3174
|
+
Fluent::Config::Element.new('buffer', 'tag,time,indexformat', {
|
3175
|
+
'chunk_keys' => ['tag', 'time', 'indexformat'],
|
3176
|
+
'timekey' => 3600,
|
3177
|
+
}, [])
|
3178
|
+
]
|
3179
|
+
))
|
3180
|
+
time = Time.parse Date.today.iso8601
|
3181
|
+
logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
|
3182
|
+
stub_elastic
|
3183
|
+
driver.run(default_tag: 'test') do
|
3184
|
+
driver.feed(time.to_i, sample_record.merge('indexformat' => '%Y.%m'))
|
3185
|
+
end
|
3186
|
+
assert_equal(logstash_index, index_cmds.first['index']['_index'])
|
3187
|
+
end
|
3188
|
+
end
|
3189
|
+
|
3124
3190
|
class HostnamePlaceholders < self
|
3125
3191
|
def test_writes_to_extracted_host
|
3126
3192
|
driver.configure("host ${tag}\n")
|
@@ -118,6 +118,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
118
118
|
end
|
119
119
|
|
120
120
|
test 'configure compression' do
|
121
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
122
|
+
|
121
123
|
config = %{
|
122
124
|
compression_level best_compression
|
123
125
|
}
|
@@ -127,6 +129,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
127
129
|
end
|
128
130
|
|
129
131
|
test 'check compression strategy' do
|
132
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
133
|
+
|
130
134
|
config = %{
|
131
135
|
compression_level best_speed
|
132
136
|
}
|
@@ -136,6 +140,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
136
140
|
end
|
137
141
|
|
138
142
|
test 'check content-encoding header with compression' do
|
143
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
144
|
+
|
139
145
|
config = %{
|
140
146
|
compression_level best_compression
|
141
147
|
}
|
@@ -145,6 +151,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
145
151
|
end
|
146
152
|
|
147
153
|
test 'check compression option is passed to transport' do
|
154
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
155
|
+
|
148
156
|
config = %{
|
149
157
|
compression_level best_compression
|
150
158
|
}
|
@@ -341,6 +349,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
341
349
|
end
|
342
350
|
|
343
351
|
def test_writes_to_default_index_with_compression
|
352
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
|
353
|
+
|
344
354
|
config = %[
|
345
355
|
compression_level default_compression
|
346
356
|
]
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.
|
4
|
+
version: 4.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-02
|
13
|
+
date: 2020-03-02 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -148,6 +148,7 @@ files:
|
|
148
148
|
- Rakefile
|
149
149
|
- appveyor.yml
|
150
150
|
- fluent-plugin-elasticsearch.gemspec
|
151
|
+
- gemfiles/Gemfile.elasticsearch.v6
|
151
152
|
- gemfiles/Gemfile.ilm
|
152
153
|
- lib/fluent/log-ext.rb
|
153
154
|
- lib/fluent/plugin/default-ilm-policy.json
|
@@ -170,6 +171,7 @@ files:
|
|
170
171
|
- test/plugin/test_elasticsearch_tls.rb
|
171
172
|
- test/plugin/test_filter_elasticsearch_genid.rb
|
172
173
|
- test/plugin/test_in_elasticsearch.rb
|
174
|
+
- test/plugin/test_oj_serializer.rb
|
173
175
|
- test/plugin/test_out_elasticsearch.rb
|
174
176
|
- test/plugin/test_out_elasticsearch_dynamic.rb
|
175
177
|
- test/plugin/test_template.json
|
@@ -206,6 +208,7 @@ test_files:
|
|
206
208
|
- test/plugin/test_elasticsearch_tls.rb
|
207
209
|
- test/plugin/test_filter_elasticsearch_genid.rb
|
208
210
|
- test/plugin/test_in_elasticsearch.rb
|
211
|
+
- test/plugin/test_oj_serializer.rb
|
209
212
|
- test/plugin/test_out_elasticsearch.rb
|
210
213
|
- test/plugin/test_out_elasticsearch_dynamic.rb
|
211
214
|
- test/plugin/test_template.json
|