fluent-plugin-opensearch 1.1.0 → 1.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/History.md +14 -0
- data/README.md +1 -1
- data/fluent-plugin-opensearch.gemspec +1 -1
- data/lib/fluent/plugin/out_opensearch_data_stream.rb +30 -24
- data/test/helper.rb +0 -1
- data/test/plugin/test_out_opensearch_data_stream.rb +65 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 578e5c63061d5f1cb3f3729dee4ea630564dc60d82f6b047c8e4b23830fdf16a
|
4
|
+
data.tar.gz: a502d46eb5a3a59c5c4f65f9009623ff32fde1de4cfda7e181437a8720bdbda1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77b3373041396440cf8debaea8fd2ab95b94df950cbfe6105f65d45cce7092a6f89bd019b38d4022051c651541d3d21918625ea1b9939f58108101eae90a027f
|
7
|
+
data.tar.gz: 2e746cd797f5d24f08dbb386ea42b9bff506831c56f9f4b92dcf99dde321b9b613c7473cbad39bd1f6d338ae6dd094a87f838141306111ab49b5a76886346918
|
data/History.md
CHANGED
@@ -2,6 +2,20 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 1.1.4
|
6
|
+
- test: remove minitest to correct misjudge of the framework by flexmock (#114)
|
7
|
+
- Add logic to write method of out_opensearch_data_stream (#109)
|
8
|
+
|
9
|
+
### 1.1.3
|
10
|
+
- Revert the behavior of passing duration second (#108)
|
11
|
+
|
12
|
+
### 1.1.2
|
13
|
+
- Check OS cluster for data streams and templates for index template creation (#106)
|
14
|
+
- out\_opensearch\_data\_stream: Don't connect to opensearch on dry-run (#105)
|
15
|
+
|
16
|
+
### 1.1.1
|
17
|
+
- Pass a value of refresh\_credentials\_interval as duration\_seconds (#78)
|
18
|
+
|
5
19
|
### 1.1.0
|
6
20
|
- Unpin `faraday` from v1, upgrade to v2.
|
7
21
|
Note that if you can't migrate other plugins from `faraday` v1 yet, need to keep
|
data/README.md
CHANGED
@@ -1606,7 +1606,7 @@ There are usually a few feature requests, tagged [Easy](https://github.com/fluen
|
|
1606
1606
|
|
1607
1607
|
Pull Requests are welcomed.
|
1608
1608
|
|
1609
|
-
|
1609
|
+
Before sending a pull request or reporting an issue, please read [the contribution guideline](CONTRIBUTING.md).
|
1610
1610
|
|
1611
1611
|
## Running tests
|
1612
1612
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-opensearch'
|
6
|
-
s.version = '1.1.
|
6
|
+
s.version = '1.1.4'
|
7
7
|
s.authors = ['Hiroshi Hatake']
|
8
8
|
s.email = ['cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Opensearch output plugin for Fluent event collector}
|
@@ -69,23 +69,26 @@ module Fluent::Plugin
|
|
69
69
|
|
70
70
|
def create_index_template(datastream_name, template_name, host = nil)
|
71
71
|
# Create index template from file
|
72
|
-
if
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
72
|
+
if !dry_run?
|
73
|
+
if @template_file
|
74
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
75
|
+
template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
|
76
|
+
else # Create default index template
|
77
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
78
|
+
body = {
|
79
|
+
"index_patterns" => ["#{datastream_name}*"],
|
80
|
+
"data_stream" => {},
|
81
|
+
}
|
82
|
+
|
83
|
+
params = {
|
84
|
+
name: template_name,
|
85
|
+
body: body
|
86
|
+
}
|
87
|
+
retry_operate(@max_retry_putting_template,
|
88
|
+
@fail_on_putting_template_retry_exceed,
|
89
|
+
@catch_transport_exception_on_retry) do
|
90
|
+
client(host).indices.put_index_template(params)
|
91
|
+
end
|
89
92
|
end
|
90
93
|
end
|
91
94
|
end
|
@@ -159,13 +162,10 @@ module Fluent::Plugin
|
|
159
162
|
end
|
160
163
|
data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
|
161
164
|
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
rescue => e
|
167
|
-
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
168
|
-
end
|
165
|
+
begin
|
166
|
+
create_index_template(data_stream_name, data_stream_template_name, host)
|
167
|
+
rescue => e
|
168
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
169
169
|
end
|
170
170
|
end
|
171
171
|
|
@@ -187,6 +187,12 @@ module Fluent::Plugin
|
|
187
187
|
dt = Time.at(time).to_datetime
|
188
188
|
end
|
189
189
|
record.merge!({"@timestamp" => dt.iso8601(@time_precision)})
|
190
|
+
if @include_tag_key
|
191
|
+
record[@tag_key] = tag
|
192
|
+
end
|
193
|
+
if @remove_keys
|
194
|
+
@remove_keys.each { |key| record.delete(key) }
|
195
|
+
end
|
190
196
|
bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
|
191
197
|
rescue => e
|
192
198
|
emit_error_label_event do
|
data/test/helper.rb
CHANGED
@@ -678,4 +678,69 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
|
|
678
678
|
assert(index_cmds[1].has_key? '@timestamp')
|
679
679
|
end
|
680
680
|
|
681
|
+
def test_record_with_include_tag_key
|
682
|
+
stub_default
|
683
|
+
stub_bulk_feed
|
684
|
+
stub_default
|
685
|
+
stub_bulk_feed
|
686
|
+
conf = config_element(
|
687
|
+
'ROOT', '', {
|
688
|
+
'@type' => OPENSEARCH_DATA_STREAM_TYPE,
|
689
|
+
'data_stream_name' => 'foo',
|
690
|
+
'data_stream_template_name' => 'foo_tpl',
|
691
|
+
'include_tag_key' => true,
|
692
|
+
'tag_key' => 'test_tag'
|
693
|
+
})
|
694
|
+
record = {
|
695
|
+
'message' => 'Sample Record'
|
696
|
+
}
|
697
|
+
driver(conf).run(default_tag: 'test') do
|
698
|
+
driver.feed(record)
|
699
|
+
end
|
700
|
+
assert(index_cmds[1].has_key?('test_tag'))
|
701
|
+
end
|
702
|
+
|
703
|
+
def test_record_without_include_tag_key
|
704
|
+
stub_default
|
705
|
+
stub_bulk_feed
|
706
|
+
stub_default
|
707
|
+
stub_bulk_feed
|
708
|
+
conf = config_element(
|
709
|
+
'ROOT', '', {
|
710
|
+
'@type' => OPENSEARCH_DATA_STREAM_TYPE,
|
711
|
+
'data_stream_name' => 'foo',
|
712
|
+
'data_stream_template_name' => 'foo_tpl',
|
713
|
+
'include_tag_key' => false
|
714
|
+
})
|
715
|
+
record = {
|
716
|
+
'message' => 'Sample Record'
|
717
|
+
}
|
718
|
+
driver(conf).run(default_tag: 'test') do
|
719
|
+
driver.feed(record)
|
720
|
+
end
|
721
|
+
assert(!index_cmds[1].has_key?('test'))
|
722
|
+
end
|
723
|
+
|
724
|
+
def test_record_with_remove_keys
|
725
|
+
stub_default
|
726
|
+
stub_bulk_feed
|
727
|
+
stub_default
|
728
|
+
stub_bulk_feed
|
729
|
+
conf = config_element(
|
730
|
+
'ROOT', '', {
|
731
|
+
'@type' => OPENSEARCH_DATA_STREAM_TYPE,
|
732
|
+
'data_stream_name' => 'foo',
|
733
|
+
'data_stream_template_name' => 'foo_tpl',
|
734
|
+
'remove_keys' => 'remove_me'
|
735
|
+
})
|
736
|
+
record = {
|
737
|
+
'message' => 'Sample Record',
|
738
|
+
'remove_me' => 'foo'
|
739
|
+
}
|
740
|
+
driver(conf).run(default_tag: 'test') do
|
741
|
+
driver.feed(record)
|
742
|
+
end
|
743
|
+
assert(!index_cmds[1].has_key?('remove_me'))
|
744
|
+
end
|
745
|
+
|
681
746
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-opensearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-08-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -270,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
270
270
|
- !ruby/object:Gem::Version
|
271
271
|
version: '0'
|
272
272
|
requirements: []
|
273
|
-
rubygems_version: 3.
|
273
|
+
rubygems_version: 3.2.32
|
274
274
|
signing_key:
|
275
275
|
specification_version: 4
|
276
276
|
summary: Opensearch output plugin for Fluent event collector
|