fluent-plugin-opensearch 1.1.0 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1b8cc79e0b3618ac9af026312e4d73784ea6bc939e944f03bcae487a239a3106
|
4
|
+
data.tar.gz: 67a5c76a80dc1f00bb13febd1feac5ab029393934d8f2e60b74f6af9c96e03d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2dadfc649ab1f9836031963ee85cdfc739029f71f35bdbd44b55f61c35351ffe1f84b64f118d6adf16ae866ebb9bc479fdcce3d71d2d0f83baba2f4fba013e80
|
7
|
+
data.tar.gz: e3a1a7df140ec59197d231b86a291dd1686d47ab2817da3983e3e9931b5e71e5994b138a6b2f10e1191a9f4deaba458c5cd221e541e66cbd534f533de85ec633
|
data/History.md
CHANGED
@@ -2,6 +2,13 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 1.1.2
|
6
|
+
- Check OS cluster for data streams and templates for index template creation (#106)
|
7
|
+
- out\_opensearch\_data\_stream: Don't connect to opensearch on dry-run (#105)
|
8
|
+
|
9
|
+
### 1.1.1
|
10
|
+
- Pass a value of refresh\_credentials\_interval as duration\_seconds (#78)
|
11
|
+
|
5
12
|
### 1.1.0
|
6
13
|
- Unpin `faraday` from v1, upgrade to v2.
|
7
14
|
Note that if you can't migrate other plugins from `faraday` v1 yet, need to keep
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-opensearch'
|
6
|
-
s.version = '1.1.
|
6
|
+
s.version = '1.1.2'
|
7
7
|
s.authors = ['Hiroshi Hatake']
|
8
8
|
s.email = ['cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Opensearch output plugin for Fluent event collector}
|
@@ -83,6 +83,7 @@ module Fluent::Plugin
|
|
83
83
|
attr_reader :template_names
|
84
84
|
attr_reader :ssl_version_options
|
85
85
|
attr_reader :compressable_connection
|
86
|
+
attr_reader :duration_seconds
|
86
87
|
|
87
88
|
helpers :event_emitter, :compat_parameters, :record_accessor, :timer
|
88
89
|
|
@@ -94,6 +95,7 @@ module Fluent::Plugin
|
|
94
95
|
DEFAULT_RELOAD_AFTER = -1
|
95
96
|
DEFAULT_TARGET_BULK_BYTES = -1
|
96
97
|
DEFAULT_POLICY_ID = "logstash-policy"
|
98
|
+
DEFAULT_DURATION = "5h"
|
97
99
|
|
98
100
|
config_param :host, :string, :default => 'localhost'
|
99
101
|
config_param :port, :integer, :default => 9200
|
@@ -195,7 +197,7 @@ module Fluent::Plugin
|
|
195
197
|
config_param :assume_role_session_name, :string, :default => "fluentd"
|
196
198
|
config_param :assume_role_web_identity_token_file, :string, :default => nil
|
197
199
|
config_param :sts_credentials_region, :string, :default => nil
|
198
|
-
config_param :refresh_credentials_interval, :time, :default =>
|
200
|
+
config_param :refresh_credentials_interval, :time, :default => DEFAULT_DURATION
|
199
201
|
config_param :aws_service_name, :enum, list: [:es, :aoss], :default => :es
|
200
202
|
end
|
201
203
|
|
@@ -211,6 +213,8 @@ module Fluent::Plugin
|
|
211
213
|
|
212
214
|
def initialize
|
213
215
|
super
|
216
|
+
|
217
|
+
@duration_seconds = Fluent::Config.time_value(DEFAULT_DURATION)
|
214
218
|
end
|
215
219
|
|
216
220
|
######################################################################################################
|
@@ -238,13 +242,15 @@ module Fluent::Plugin
|
|
238
242
|
credentials = Aws::AssumeRoleCredentials.new({
|
239
243
|
role_arn: conf[:assume_role_arn],
|
240
244
|
role_session_name: conf[:assume_role_session_name],
|
241
|
-
region: sts_creds_region(conf)
|
245
|
+
region: sts_creds_region(conf),
|
246
|
+
duration_seconds: @duration_seconds
|
242
247
|
}).credentials
|
243
248
|
else
|
244
249
|
credentials = Aws::AssumeRoleWebIdentityCredentials.new({
|
245
250
|
role_arn: conf[:assume_role_arn],
|
246
251
|
web_identity_token_file: conf[:assume_role_web_identity_token_file],
|
247
|
-
region: sts_creds_region(conf)
|
252
|
+
region: sts_creds_region(conf),
|
253
|
+
duration_seconds: @duration_seconds
|
248
254
|
}).credentials
|
249
255
|
end
|
250
256
|
end
|
@@ -345,7 +351,18 @@ module Fluent::Plugin
|
|
345
351
|
@_aws_credentials = aws_credentials(@endpoint)
|
346
352
|
|
347
353
|
if @endpoint.refresh_credentials_interval
|
348
|
-
|
354
|
+
@duration_seconds = Fluent::Config.time_value(@endpoint.refresh_credentials_interval)
|
355
|
+
# 60 * 60 * 12 = 12 hours
|
356
|
+
if @duration_seconds > 43200
|
357
|
+
raise Fluent::ConfigError, "Maximum duration is 12 hours."
|
358
|
+
end
|
359
|
+
|
360
|
+
# 60 * 15 = 15 minutes
|
361
|
+
if @duration_seconds < 900
|
362
|
+
raise Fluent::ConfigError, "Minimum duration is 15 minutes."
|
363
|
+
end
|
364
|
+
|
365
|
+
timer_execute(:out_opensearch_expire_credentials, @duration_seconds) do
|
349
366
|
log.debug('Recreate the AWS credentials')
|
350
367
|
|
351
368
|
@credential_mutex.synchronize do
|
@@ -69,23 +69,26 @@ module Fluent::Plugin
|
|
69
69
|
|
70
70
|
def create_index_template(datastream_name, template_name, host = nil)
|
71
71
|
# Create index template from file
|
72
|
-
if
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
72
|
+
if !dry_run?
|
73
|
+
if @template_file
|
74
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
75
|
+
template_installation_actual(template_name, @customize_template, @application_name, datastream_name, host)
|
76
|
+
else # Create default index template
|
77
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
78
|
+
body = {
|
79
|
+
"index_patterns" => ["#{datastream_name}*"],
|
80
|
+
"data_stream" => {},
|
81
|
+
}
|
82
|
+
|
83
|
+
params = {
|
84
|
+
name: template_name,
|
85
|
+
body: body
|
86
|
+
}
|
87
|
+
retry_operate(@max_retry_putting_template,
|
88
|
+
@fail_on_putting_template_retry_exceed,
|
89
|
+
@catch_transport_exception_on_retry) do
|
90
|
+
client(host).indices.put_index_template(params)
|
91
|
+
end
|
89
92
|
end
|
90
93
|
end
|
91
94
|
end
|
@@ -159,13 +162,10 @@ module Fluent::Plugin
|
|
159
162
|
end
|
160
163
|
data_stream_name = extract_placeholders(@data_stream_name, chunk).downcase
|
161
164
|
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk).downcase
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
rescue => e
|
167
|
-
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
168
|
-
end
|
165
|
+
begin
|
166
|
+
create_index_template(data_stream_name, data_stream_template_name, host)
|
167
|
+
rescue => e
|
168
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
169
169
|
end
|
170
170
|
end
|
171
171
|
|
@@ -300,6 +300,7 @@ class OpenSearchOutputTest < Test::Unit::TestCase
|
|
300
300
|
'region' => "local",
|
301
301
|
'access_key_id' => 'YOUR_AWESOME_KEY',
|
302
302
|
'secret_access_key' => 'YOUR_AWESOME_SECRET',
|
303
|
+
'refresh_credentials_interval' => '10h'
|
303
304
|
}, []),
|
304
305
|
Fluent::Config::Element.new('buffer', 'tag', {}, [])
|
305
306
|
|
@@ -316,6 +317,8 @@ class OpenSearchOutputTest < Test::Unit::TestCase
|
|
316
317
|
assert_nil instance.endpoint.assume_role_web_identity_token_file
|
317
318
|
assert_nil instance.endpoint.sts_credentials_region
|
318
319
|
assert_equal :es, instance.endpoint.aws_service_name
|
320
|
+
assert_equal 36000, instance.endpoint.refresh_credentials_interval
|
321
|
+
assert_equal 36000, instance.duration_seconds
|
319
322
|
end
|
320
323
|
|
321
324
|
data("OpenSearch Service" => [:es, 'es'],
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-opensearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hiroshi Hatake
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -270,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
270
270
|
- !ruby/object:Gem::Version
|
271
271
|
version: '0'
|
272
272
|
requirements: []
|
273
|
-
rubygems_version: 3.
|
273
|
+
rubygems_version: 3.2.32
|
274
274
|
signing_key:
|
275
275
|
specification_version: 4
|
276
276
|
summary: Opensearch output plugin for Fluent event collector
|