test-opensearch 0.0.0.2 → 0.0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-opensearch.gemspec +1 -1
- data/lib/fluent/plugin/in_opensearch.rb +84 -26
- data/test/plugin/test_in_opensearch.rb +7 -7
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4e56816c3f750e8e813fbcf64e6589ca799a2b94f2b00a07ef0ae8395a4da14b
|
4
|
+
data.tar.gz: 0340bdd6b8a12e2ed032437484656f61b37c5cca2355bc01cadd408225619b90
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b5cabd951822174fa8cf36902279bcf41db4a3a839b783ccdc072f4dfb6816c40d502261053343d71df9e5a1d26e3d330adfcbae839bbaa9b3974f576f226d11
|
7
|
+
data.tar.gz: 58305a974872fd798841bd9df1dccf0099c50282e6bab96d8dcd3155e0ad05c0796c7e856bec379d643c6dad03caf2a0057577a3981734f4bc0b2c3100df0bac
|
@@ -26,9 +26,10 @@
|
|
26
26
|
|
27
27
|
require 'opensearch'
|
28
28
|
|
29
|
-
|
29
|
+
require 'faraday/excon'
|
30
30
|
require 'fluent/log-ext'
|
31
31
|
require 'fluent/plugin/input'
|
32
|
+
require 'fluent/plugin_helper'
|
32
33
|
require_relative 'opensearch_constants'
|
33
34
|
|
34
35
|
module Fluent::Plugin
|
@@ -39,7 +40,7 @@ module Fluent::Plugin
|
|
39
40
|
DEFAULT_STORAGE_TYPE = 'local'
|
40
41
|
METADATA = "@metadata".freeze
|
41
42
|
|
42
|
-
helpers :timer, :thread
|
43
|
+
helpers :timer, :thread, :retry_state
|
43
44
|
|
44
45
|
Fluent::Plugin.register_input('opensearch', self)
|
45
46
|
|
@@ -80,7 +81,24 @@ module Fluent::Plugin
|
|
80
81
|
config_param :docinfo_fields, :array, :default => ['_index', '_type', '_id']
|
81
82
|
config_param :docinfo_target, :string, :default => METADATA
|
82
83
|
config_param :docinfo, :bool, :default => false
|
83
|
-
config_param :
|
84
|
+
config_param :check_connection, :bool, :default => true
|
85
|
+
config_param :retry_forever, :bool, default: true, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry forever.'
|
86
|
+
config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry to flush while failing, until plugin discards buffer chunks.'
|
87
|
+
# 72hours == 17 times with exponential backoff (not to change default behavior)
|
88
|
+
config_param :retry_max_times, :integer, default: nil, desc: 'The maximum number of times to retry to flush while failing.'
|
89
|
+
config_param :retry_secondary_threshold, :float, default: 0.8, desc: 'ratio of retry_timeout to switch to use secondary while failing.'
|
90
|
+
# exponential backoff sequence will be initialized at the time of this threshold
|
91
|
+
config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :periodic
|
92
|
+
### Periodic -> fixed :retry_wait
|
93
|
+
### Exponential backoff: k is number of retry times
|
94
|
+
# c: constant factor, @retry_wait
|
95
|
+
# b: base factor, @retry_exponential_backoff_base
|
96
|
+
# k: times
|
97
|
+
# total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
|
98
|
+
config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
|
99
|
+
config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
|
100
|
+
config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
|
101
|
+
config_param :retry_randomize, :bool, default: false, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
|
84
102
|
|
85
103
|
include Fluent::Plugin::OpenSearchConstants
|
86
104
|
|
@@ -93,6 +111,7 @@ module Fluent::Plugin
|
|
93
111
|
|
94
112
|
@timestamp_parser = create_time_parser
|
95
113
|
@backend_options = backend_options
|
114
|
+
@retry = retry_state(@retry_randomize)
|
96
115
|
|
97
116
|
raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
|
98
117
|
|
@@ -139,6 +158,15 @@ module Fluent::Plugin
|
|
139
158
|
raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
|
140
159
|
end
|
141
160
|
|
161
|
+
def retry_state(randomize)
|
162
|
+
retry_state_create(
|
163
|
+
:input_retries,@retry_type,@retry_type,@retry_timeout,
|
164
|
+
forever: @retry_forever,max_steps: @retry_max_times,
|
165
|
+
max_interval: @retry_max_interval,backoff_base: @retry_exponential_backoff_base,
|
166
|
+
randomize: randomize
|
167
|
+
)
|
168
|
+
end
|
169
|
+
|
142
170
|
def get_escaped_userinfo(host_str)
|
143
171
|
if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
|
144
172
|
m["scheme"] +
|
@@ -177,12 +205,33 @@ module Fluent::Plugin
|
|
177
205
|
host.merge!(user: @user, password: @password) if !host[:user] && @user
|
178
206
|
host.merge!(path: @path) if !host[:path] && @path
|
179
207
|
end
|
180
|
-
|
208
|
+
live_hosts = hosts.select { |host| reachable_host?(host) }
|
181
209
|
{
|
182
|
-
hosts:
|
210
|
+
hosts: live_hosts
|
183
211
|
}
|
184
212
|
end
|
185
213
|
|
214
|
+
def reachable_host?(host)
|
215
|
+
if @check_connection
|
216
|
+
client = OpenSearch::Client.new(
|
217
|
+
host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
|
218
|
+
user: host[:user],
|
219
|
+
password: host[:password],
|
220
|
+
reload_connections: @reload_connections,
|
221
|
+
request_timeout: @request_timeout,
|
222
|
+
resurrect_after: @resurrect_after,
|
223
|
+
reload_on_failure: @reload_on_failure,
|
224
|
+
transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
|
225
|
+
)
|
226
|
+
client.ping
|
227
|
+
else
|
228
|
+
true
|
229
|
+
end
|
230
|
+
rescue => e
|
231
|
+
log.warn "Failed to connect to #{host[:scheme]}://#{host[:host]}:#{host[:port]}: #{e.message}"
|
232
|
+
false
|
233
|
+
end
|
234
|
+
|
186
235
|
def emit_error_label_event(&block)
|
187
236
|
# If `emit_error_label_event` is specified as false, error event emittions are not occurred.
|
188
237
|
if emit_error_label_event
|
@@ -303,30 +352,39 @@ module Fluent::Plugin
|
|
303
352
|
run_slice(slice_id)
|
304
353
|
end
|
305
354
|
end
|
306
|
-
rescue
|
307
|
-
|
308
|
-
retry
|
355
|
+
rescue => e
|
356
|
+
@retry.step
|
357
|
+
# Check if the retry limit has been reached
|
358
|
+
if @retry.limit?
|
359
|
+
msg = "Hit limit for retries."
|
360
|
+
log.warn msg, retry_times: @retry.steps, error: e.message
|
361
|
+
break
|
362
|
+
else
|
363
|
+
msg = "failed to connect or search."
|
364
|
+
log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: error)
|
365
|
+
sleep(@retry.next_time - Time.now)
|
366
|
+
retry
|
367
|
+
end
|
309
368
|
end
|
310
369
|
|
311
370
|
def run_slice(slice_id=nil)
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
# clear_scroll(scroll_id)
|
371
|
+
slice_query = @base_query
|
372
|
+
slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @num_slices}) unless slice_id.nil?
|
373
|
+
result = client.search(@options.merge(:body => Yajl.dump(slice_query) ))
|
374
|
+
es = Fluent::MultiEventStream.new
|
375
|
+
|
376
|
+
result["hits"]["hits"].each {|hit| process_events(hit, es)}
|
377
|
+
has_hits = result['hits']['hits'].any?
|
378
|
+
scroll_id = result['_scroll_id']
|
379
|
+
|
380
|
+
while has_hits && scroll_id
|
381
|
+
result = process_next_scroll_request(es, scroll_id)
|
382
|
+
has_hits = result['has_hits']
|
383
|
+
scroll_id = result['_scroll_id']
|
384
|
+
end
|
385
|
+
|
386
|
+
router.emit_stream(@tag, es)
|
387
|
+
clear_scroll(scroll_id)
|
330
388
|
end
|
331
389
|
|
332
390
|
def clear_scroll(scroll_id)
|
@@ -39,7 +39,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
|
|
39
39
|
CONFIG = %[
|
40
40
|
tag raw.opensearch
|
41
41
|
interval 2
|
42
|
-
|
42
|
+
check_connection false
|
43
43
|
]
|
44
44
|
|
45
45
|
def setup
|
@@ -191,7 +191,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
|
|
191
191
|
user john
|
192
192
|
password doe
|
193
193
|
tag raw.opensearch
|
194
|
-
|
194
|
+
check_connection false
|
195
195
|
}
|
196
196
|
instance = driver(config).instance
|
197
197
|
|
@@ -230,7 +230,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
|
|
230
230
|
user john
|
231
231
|
password doe
|
232
232
|
tag raw.opensearch
|
233
|
-
|
233
|
+
check_connection false
|
234
234
|
}
|
235
235
|
instance = driver(config).instance
|
236
236
|
|
@@ -252,7 +252,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
|
|
252
252
|
user %{j+hn}
|
253
253
|
password %{d@e}
|
254
254
|
tag raw.opensearch
|
255
|
-
|
255
|
+
check_connection false
|
256
256
|
}
|
257
257
|
instance = driver(config).instance
|
258
258
|
|
@@ -275,7 +275,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
|
|
275
275
|
path /es/
|
276
276
|
port 123
|
277
277
|
tag raw.opensearch
|
278
|
-
|
278
|
+
check_connection false
|
279
279
|
}
|
280
280
|
instance = driver(config).instance
|
281
281
|
|
@@ -300,7 +300,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
|
|
300
300
|
user default_user
|
301
301
|
password default_password
|
302
302
|
tag raw.opensearch
|
303
|
-
|
303
|
+
check_connection false
|
304
304
|
}
|
305
305
|
instance = driver(config).instance
|
306
306
|
|
@@ -329,7 +329,7 @@ class OpenSearchInputTest < Test::Unit::TestCase
|
|
329
329
|
user default_user
|
330
330
|
password default_password
|
331
331
|
tag raw.opensearch
|
332
|
-
|
332
|
+
check_connection false
|
333
333
|
}
|
334
334
|
instance = driver(config).instance
|
335
335
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: test-opensearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.0.
|
4
|
+
version: 0.0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oleh
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-04-
|
11
|
+
date: 2024-04-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|