fluent-plugin-opensearch 1.1.2 → 1.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/History.md +7 -0
- data/README.md +1 -1
- data/fluent-plugin-opensearch.gemspec +1 -1
- data/lib/fluent/plugin/out_opensearch.rb +4 -21
- data/lib/fluent/plugin/out_opensearch_data_stream.rb +6 -0
- data/test/helper.rb +0 -1
- data/test/plugin/test_out_opensearch.rb +0 -3
- data/test/plugin/test_out_opensearch_data_stream.rb +65 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 578e5c63061d5f1cb3f3729dee4ea630564dc60d82f6b047c8e4b23830fdf16a
|
4
|
+
data.tar.gz: a502d46eb5a3a59c5c4f65f9009623ff32fde1de4cfda7e181437a8720bdbda1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77b3373041396440cf8debaea8fd2ab95b94df950cbfe6105f65d45cce7092a6f89bd019b38d4022051c651541d3d21918625ea1b9939f58108101eae90a027f
|
7
|
+
data.tar.gz: 2e746cd797f5d24f08dbb386ea42b9bff506831c56f9f4b92dcf99dde321b9b613c7473cbad39bd1f6d338ae6dd094a87f838141306111ab49b5a76886346918
|
data/History.md
CHANGED
@@ -2,6 +2,13 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 1.1.4
|
6
|
+
- test: remove minitest to correct misjudge of the framework by flexmock (#114)
|
7
|
+
- Add logic to write method of out_opensearch_data_stream (#109)
|
8
|
+
|
9
|
+
### 1.1.3
|
10
|
+
- Revert the behavior of passing duration second (#108)
|
11
|
+
|
5
12
|
### 1.1.2
|
6
13
|
- Check OS cluster for data streams and templates for index template creation (#106)
|
7
14
|
- out\_opensearch\_data\_stream: Don't connect to opensearch on dry-run (#105)
|
data/README.md
CHANGED
@@ -1606,7 +1606,7 @@ There are usually a few feature requests, tagged [Easy](https://github.com/fluen
|
|
1606
1606
|
|
1607
1607
|
Pull Requests are welcomed.
|
1608
1608
|
|
1609
|
-
|
1609
|
+
Before sending a pull request or reporting an issue, please read [the contribution guideline](CONTRIBUTING.md).
|
1610
1610
|
|
1611
1611
|
## Running tests
|
1612
1612
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-opensearch'
|
6
|
-
s.version = '1.1.
|
6
|
+
s.version = '1.1.4'
|
7
7
|
s.authors = ['Hiroshi Hatake']
|
8
8
|
s.email = ['cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Opensearch output plugin for Fluent event collector}
|
@@ -83,7 +83,6 @@ module Fluent::Plugin
|
|
83
83
|
attr_reader :template_names
|
84
84
|
attr_reader :ssl_version_options
|
85
85
|
attr_reader :compressable_connection
|
86
|
-
attr_reader :duration_seconds
|
87
86
|
|
88
87
|
helpers :event_emitter, :compat_parameters, :record_accessor, :timer
|
89
88
|
|
@@ -95,7 +94,6 @@ module Fluent::Plugin
|
|
95
94
|
DEFAULT_RELOAD_AFTER = -1
|
96
95
|
DEFAULT_TARGET_BULK_BYTES = -1
|
97
96
|
DEFAULT_POLICY_ID = "logstash-policy"
|
98
|
-
DEFAULT_DURATION = "5h"
|
99
97
|
|
100
98
|
config_param :host, :string, :default => 'localhost'
|
101
99
|
config_param :port, :integer, :default => 9200
|
@@ -197,7 +195,7 @@ module Fluent::Plugin
|
|
197
195
|
config_param :assume_role_session_name, :string, :default => "fluentd"
|
198
196
|
config_param :assume_role_web_identity_token_file, :string, :default => nil
|
199
197
|
config_param :sts_credentials_region, :string, :default => nil
|
200
|
-
config_param :refresh_credentials_interval, :time, :default =>
|
198
|
+
config_param :refresh_credentials_interval, :time, :default => "5h"
|
201
199
|
config_param :aws_service_name, :enum, list: [:es, :aoss], :default => :es
|
202
200
|
end
|
203
201
|
|
@@ -213,8 +211,6 @@ module Fluent::Plugin
|
|
213
211
|
|
214
212
|
def initialize
|
215
213
|
super
|
216
|
-
|
217
|
-
@duration_seconds = Fluent::Config.time_value(DEFAULT_DURATION)
|
218
214
|
end
|
219
215
|
|
220
216
|
######################################################################################################
|
@@ -242,15 +238,13 @@ module Fluent::Plugin
|
|
242
238
|
credentials = Aws::AssumeRoleCredentials.new({
|
243
239
|
role_arn: conf[:assume_role_arn],
|
244
240
|
role_session_name: conf[:assume_role_session_name],
|
245
|
-
region: sts_creds_region(conf)
|
246
|
-
duration_seconds: @duration_seconds
|
241
|
+
region: sts_creds_region(conf)
|
247
242
|
}).credentials
|
248
243
|
else
|
249
244
|
credentials = Aws::AssumeRoleWebIdentityCredentials.new({
|
250
245
|
role_arn: conf[:assume_role_arn],
|
251
246
|
web_identity_token_file: conf[:assume_role_web_identity_token_file],
|
252
|
-
region: sts_creds_region(conf)
|
253
|
-
duration_seconds: @duration_seconds
|
247
|
+
region: sts_creds_region(conf)
|
254
248
|
}).credentials
|
255
249
|
end
|
256
250
|
end
|
@@ -351,18 +345,7 @@ module Fluent::Plugin
|
|
351
345
|
@_aws_credentials = aws_credentials(@endpoint)
|
352
346
|
|
353
347
|
if @endpoint.refresh_credentials_interval
|
354
|
-
|
355
|
-
# 60 * 60 * 12 = 12 hours
|
356
|
-
if @duration_seconds > 43200
|
357
|
-
raise Fluent::ConfigError, "Maximum duration is 12 hours."
|
358
|
-
end
|
359
|
-
|
360
|
-
# 60 * 15 = 15 minutes
|
361
|
-
if @duration_seconds < 900
|
362
|
-
raise Fluent::ConfigError, "Minimum duration is 15 minutes."
|
363
|
-
end
|
364
|
-
|
365
|
-
timer_execute(:out_opensearch_expire_credentials, @duration_seconds) do
|
348
|
+
timer_execute(:out_opensearch_expire_credentials, @endpoint.refresh_credentials_interval) do
|
366
349
|
log.debug('Recreate the AWS credentials')
|
367
350
|
|
368
351
|
@credential_mutex.synchronize do
|
@@ -187,6 +187,12 @@ module Fluent::Plugin
|
|
187
187
|
dt = Time.at(time).to_datetime
|
188
188
|
end
|
189
189
|
record.merge!({"@timestamp" => dt.iso8601(@time_precision)})
|
190
|
+
if @include_tag_key
|
191
|
+
record[@tag_key] = tag
|
192
|
+
end
|
193
|
+
if @remove_keys
|
194
|
+
@remove_keys.each { |key| record.delete(key) }
|
195
|
+
end
|
190
196
|
bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
|
191
197
|
rescue => e
|
192
198
|
emit_error_label_event do
|
data/test/helper.rb
CHANGED
@@ -300,7 +300,6 @@ class OpenSearchOutputTest < Test::Unit::TestCase
|
|
300
300
|
'region' => "local",
|
301
301
|
'access_key_id' => 'YOUR_AWESOME_KEY',
|
302
302
|
'secret_access_key' => 'YOUR_AWESOME_SECRET',
|
303
|
-
'refresh_credentials_interval' => '10h'
|
304
303
|
}, []),
|
305
304
|
Fluent::Config::Element.new('buffer', 'tag', {}, [])
|
306
305
|
|
@@ -317,8 +316,6 @@ class OpenSearchOutputTest < Test::Unit::TestCase
|
|
317
316
|
assert_nil instance.endpoint.assume_role_web_identity_token_file
|
318
317
|
assert_nil instance.endpoint.sts_credentials_region
|
319
318
|
assert_equal :es, instance.endpoint.aws_service_name
|
320
|
-
assert_equal 36000, instance.endpoint.refresh_credentials_interval
|
321
|
-
assert_equal 36000, instance.duration_seconds
|
322
319
|
end
|
323
320
|
|
324
321
|
data("OpenSearch Service" => [:es, 'es'],
|
@@ -678,4 +678,69 @@ class OpenSearchOutputDataStreamTest < Test::Unit::TestCase
|
|
678
678
|
assert(index_cmds[1].has_key? '@timestamp')
|
679
679
|
end
|
680
680
|
|
681
|
+
def test_record_with_include_tag_key
|
682
|
+
stub_default
|
683
|
+
stub_bulk_feed
|
684
|
+
stub_default
|
685
|
+
stub_bulk_feed
|
686
|
+
conf = config_element(
|
687
|
+
'ROOT', '', {
|
688
|
+
'@type' => OPENSEARCH_DATA_STREAM_TYPE,
|
689
|
+
'data_stream_name' => 'foo',
|
690
|
+
'data_stream_template_name' => 'foo_tpl',
|
691
|
+
'include_tag_key' => true,
|
692
|
+
'tag_key' => 'test_tag'
|
693
|
+
})
|
694
|
+
record = {
|
695
|
+
'message' => 'Sample Record'
|
696
|
+
}
|
697
|
+
driver(conf).run(default_tag: 'test') do
|
698
|
+
driver.feed(record)
|
699
|
+
end
|
700
|
+
assert(index_cmds[1].has_key?('test_tag'))
|
701
|
+
end
|
702
|
+
|
703
|
+
def test_record_without_include_tag_key
|
704
|
+
stub_default
|
705
|
+
stub_bulk_feed
|
706
|
+
stub_default
|
707
|
+
stub_bulk_feed
|
708
|
+
conf = config_element(
|
709
|
+
'ROOT', '', {
|
710
|
+
'@type' => OPENSEARCH_DATA_STREAM_TYPE,
|
711
|
+
'data_stream_name' => 'foo',
|
712
|
+
'data_stream_template_name' => 'foo_tpl',
|
713
|
+
'include_tag_key' => false
|
714
|
+
})
|
715
|
+
record = {
|
716
|
+
'message' => 'Sample Record'
|
717
|
+
}
|
718
|
+
driver(conf).run(default_tag: 'test') do
|
719
|
+
driver.feed(record)
|
720
|
+
end
|
721
|
+
assert(!index_cmds[1].has_key?('test'))
|
722
|
+
end
|
723
|
+
|
724
|
+
def test_record_with_remove_keys
|
725
|
+
stub_default
|
726
|
+
stub_bulk_feed
|
727
|
+
stub_default
|
728
|
+
stub_bulk_feed
|
729
|
+
conf = config_element(
|
730
|
+
'ROOT', '', {
|
731
|
+
'@type' => OPENSEARCH_DATA_STREAM_TYPE,
|
732
|
+
'data_stream_name' => 'foo',
|
733
|
+
'data_stream_template_name' => 'foo_tpl',
|
734
|
+
'remove_keys' => 'remove_me'
|
735
|
+
})
|
736
|
+
record = {
|
737
|
+
'message' => 'Sample Record',
|
738
|
+
'remove_me' => 'foo'
|
739
|
+
}
|
740
|
+
driver(conf).run(default_tag: 'test') do
|
741
|
+
driver.feed(record)
|
742
|
+
end
|
743
|
+
assert(!index_cmds[1].has_key?('remove_me'))
|
744
|
+
end
|
745
|
+
|
681
746
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-opensearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-08-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|