test-input-opensearch 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.coveralls.yml +1 -0
- data/.editorconfig +9 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +29 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +9 -0
- data/.github/workflows/coverage.yaml +22 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/.gitignore +18 -0
- data/CONTRIBUTING.md +24 -0
- data/Gemfile +10 -0
- data/History.md +67 -0
- data/LICENSE.txt +201 -0
- data/README.OpenSearchGenID.md +116 -0
- data/README.OpenSearchInput.md +396 -0
- data/README.Troubleshooting.md +482 -0
- data/README.md +1622 -0
- data/Rakefile +37 -0
- data/fluent-plugin-opensearch.gemspec +39 -0
- data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
- data/lib/fluent/log-ext.rb +64 -0
- data/lib/fluent/plugin/filter_opensearch_genid.rb +103 -0
- data/lib/fluent/plugin/in_opensearch.rb +441 -0
- data/lib/fluent/plugin/oj_serializer.rb +48 -0
- data/lib/fluent/plugin/opensearch_constants.rb +39 -0
- data/lib/fluent/plugin/opensearch_error.rb +31 -0
- data/lib/fluent/plugin/opensearch_error_handler.rb +182 -0
- data/lib/fluent/plugin/opensearch_fallback_selector.rb +36 -0
- data/lib/fluent/plugin/opensearch_index_template.rb +155 -0
- data/lib/fluent/plugin/opensearch_simple_sniffer.rb +36 -0
- data/lib/fluent/plugin/opensearch_tls.rb +96 -0
- data/lib/fluent/plugin/out_opensearch.rb +1158 -0
- data/lib/fluent/plugin/out_opensearch_data_stream.rb +231 -0
- data/test/helper.rb +60 -0
- data/test/plugin/datastream_template.json +4 -0
- data/test/plugin/test_alias_template.json +9 -0
- data/test/plugin/test_filter_opensearch_genid.rb +241 -0
- data/test/plugin/test_in_opensearch.rb +500 -0
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_oj_serializer.rb +45 -0
- data/test/plugin/test_opensearch_error_handler.rb +770 -0
- data/test/plugin/test_opensearch_fallback_selector.rb +100 -0
- data/test/plugin/test_opensearch_tls.rb +171 -0
- data/test/plugin/test_out_opensearch.rb +3980 -0
- data/test/plugin/test_out_opensearch_data_stream.rb +746 -0
- data/test/plugin/test_template.json +23 -0
- data/test/test_log-ext.rb +61 -0
- metadata +291 -0
@@ -0,0 +1,441 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed to Uken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Uken Inc. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
require 'opensearch'
|
28
|
+
|
29
|
+
require 'faraday/excon'
|
30
|
+
require 'fluent/log-ext'
|
31
|
+
require 'fluent/plugin/input'
|
32
|
+
require 'fluent/plugin_helper'
|
33
|
+
require_relative 'opensearch_constants'
|
34
|
+
|
35
|
+
module Fluent::Plugin
|
36
|
+
class OpenSearchInput < Input
|
37
|
+
class UnrecoverableRequestFailure < Fluent::UnrecoverableError; end
|
38
|
+
|
39
|
+
DEFAULT_RELOAD_AFTER = -1
|
40
|
+
DEFAULT_STORAGE_TYPE = 'local'
|
41
|
+
METADATA = "@metadata".freeze
|
42
|
+
|
43
|
+
helpers :timer, :thread, :retry_state
|
44
|
+
|
45
|
+
Fluent::Plugin.register_input('opensearch', self)
|
46
|
+
|
47
|
+
config_param :tag, :string
|
48
|
+
config_param :host, :string, :default => 'localhost'
|
49
|
+
config_param :port, :integer, :default => 9200
|
50
|
+
config_param :user, :string, :default => nil
|
51
|
+
config_param :password, :string, :default => nil, :secret => true
|
52
|
+
config_param :path, :string, :default => nil
|
53
|
+
config_param :scheme, :enum, :list => [:https, :http], :default => :http
|
54
|
+
config_param :hosts, :string, :default => nil
|
55
|
+
config_param :index_name, :string, :default => "fluentd"
|
56
|
+
config_param :parse_timestamp, :bool, :default => false
|
57
|
+
config_param :timestamp_key_format, :string, :default => nil
|
58
|
+
config_param :timestamp_parse_error_tag, :string, :default => 'opensearch_plugin.input.time.error'
|
59
|
+
config_param :query, :hash, :default => {"sort" => [ "_doc" ]}
|
60
|
+
config_param :scroll, :string, :default => "1m"
|
61
|
+
config_param :size, :integer, :default => 1000
|
62
|
+
config_param :num_slices, :integer, :default => 1
|
63
|
+
config_param :interval, :size, :default => 5
|
64
|
+
config_param :repeat, :bool, :default => true
|
65
|
+
config_param :http_backend, :enum, list: [:excon, :typhoeus], :default => :excon
|
66
|
+
config_param :request_timeout, :time, :default => 5
|
67
|
+
config_param :reload_connections, :bool, :default => false
|
68
|
+
config_param :reload_on_failure, :bool, :default => false
|
69
|
+
config_param :resurrect_after, :time, :default => 60
|
70
|
+
config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
|
71
|
+
config_param :ssl_verify , :bool, :default => true
|
72
|
+
config_param :client_key, :string, :default => nil
|
73
|
+
config_param :client_cert, :string, :default => nil
|
74
|
+
config_param :client_key_pass, :string, :default => nil, :secret => true
|
75
|
+
config_param :ca_file, :string, :default => nil
|
76
|
+
config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1_2
|
77
|
+
config_param :with_transporter_log, :bool, :default => false
|
78
|
+
config_param :emit_error_label_event, :bool, :default => true
|
79
|
+
config_param :sniffer_class_name, :string, :default => nil
|
80
|
+
config_param :custom_headers, :hash, :default => {}
|
81
|
+
config_param :docinfo_fields, :array, :default => ['_index', '_type', '_id']
|
82
|
+
config_param :docinfo_target, :string, :default => METADATA
|
83
|
+
config_param :docinfo, :bool, :default => false
|
84
|
+
config_param :check_connection, :bool, :default => true
|
85
|
+
config_param :retry_forever, :bool, default: true, desc: 'If true, plugin will ignore retry_timeout and retry_max_times options and retry forever.'
|
86
|
+
config_param :retry_timeout, :time, default: 72 * 60 * 60, desc: 'The maximum seconds to retry'
|
87
|
+
# 72hours == 17 times with exponential backoff (not to change default behavior)
|
88
|
+
config_param :retry_max_times, :integer, default: 5, desc: 'The maximum number of times to retry'
|
89
|
+
# exponential backoff sequence will be initialized at the time of this threshold
|
90
|
+
config_param :retry_type, :enum, list: [:exponential_backoff, :periodic], default: :exponential_backoff
|
91
|
+
### Periodic -> fixed :retry_wait
|
92
|
+
### Exponential backoff: k is number of retry times
|
93
|
+
# c: constant factor, @retry_wait
|
94
|
+
# b: base factor, @retry_exponential_backoff_base
|
95
|
+
# k: times
|
96
|
+
# total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
|
97
|
+
config_param :retry_wait, :time, default: 5, desc: 'Seconds to wait before next retry , or constant factor of exponential backoff.'
|
98
|
+
config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
|
99
|
+
config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
|
100
|
+
config_param :retry_randomize, :bool, default: false, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
|
101
|
+
|
102
|
+
include Fluent::Plugin::OpenSearchConstants
|
103
|
+
|
104
|
+
def initialize
|
105
|
+
super
|
106
|
+
end
|
107
|
+
|
108
|
+
def configure(conf)
|
109
|
+
super
|
110
|
+
|
111
|
+
@timestamp_parser = create_time_parser
|
112
|
+
@backend_options = backend_options
|
113
|
+
@retry = nil
|
114
|
+
|
115
|
+
raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
|
116
|
+
|
117
|
+
if @user && m = @user.match(/%{(?<user>.*)}/)
|
118
|
+
@user = URI.encode_www_form_component(m["user"])
|
119
|
+
end
|
120
|
+
if @password && m = @password.match(/%{(?<password>.*)}/)
|
121
|
+
@password = URI.encode_www_form_component(m["password"])
|
122
|
+
end
|
123
|
+
|
124
|
+
@transport_logger = nil
|
125
|
+
if @with_transporter_log
|
126
|
+
@transport_logger = log
|
127
|
+
log_level = conf['@log_level'] || conf['log_level']
|
128
|
+
log.warn "Consider to specify log_level with @log_level." unless log_level
|
129
|
+
end
|
130
|
+
@current_config = nil
|
131
|
+
# Specify @sniffer_class before calling #client.
|
132
|
+
@sniffer_class = nil
|
133
|
+
begin
|
134
|
+
@sniffer_class = Object.const_get(@sniffer_class_name) if @sniffer_class_name
|
135
|
+
rescue Exception => ex
|
136
|
+
raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
|
137
|
+
end
|
138
|
+
|
139
|
+
@options = {
|
140
|
+
:index => @index_name,
|
141
|
+
:scroll => @scroll,
|
142
|
+
:size => @size
|
143
|
+
}
|
144
|
+
@base_query = @query
|
145
|
+
end
|
146
|
+
|
147
|
+
def backend_options
|
148
|
+
case @http_backend
|
149
|
+
when :excon
|
150
|
+
{ client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
|
151
|
+
when :typhoeus
|
152
|
+
require 'typhoeus'
|
153
|
+
{ sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
|
154
|
+
end
|
155
|
+
rescue LoadError => ex
|
156
|
+
log.error_backtrace(ex.backtrace)
|
157
|
+
raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
|
158
|
+
end
|
159
|
+
|
160
|
+
def retry_state(randomize)
|
161
|
+
retry_state_create(
|
162
|
+
:input_retries, @retry_type, @retry_wait, @retry_timeout,
|
163
|
+
forever: @retry_forever, max_steps: @retry_max_times,
|
164
|
+
max_interval: @retry_max_interval, backoff_base: @retry_exponential_backoff_base,
|
165
|
+
randomize: randomize
|
166
|
+
)
|
167
|
+
end
|
168
|
+
|
169
|
+
def get_escaped_userinfo(host_str)
|
170
|
+
if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
|
171
|
+
m["scheme"] +
|
172
|
+
URI.encode_www_form_component(m["user"]) +
|
173
|
+
':' +
|
174
|
+
URI.encode_www_form_component(m["password"]) +
|
175
|
+
m["path"]
|
176
|
+
else
|
177
|
+
host_str
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
def get_connection_options(con_host=nil)
|
182
|
+
|
183
|
+
hosts = if con_host || @hosts
|
184
|
+
(con_host || @hosts).split(',').map do |host_str|
|
185
|
+
# Support legacy hosts format host:port,host:port,host:port...
|
186
|
+
if host_str.match(%r{^[^:]+(\:\d+)?$})
|
187
|
+
{
|
188
|
+
host: host_str.split(':')[0],
|
189
|
+
port: (host_str.split(':')[1] || @port).to_i,
|
190
|
+
scheme: @scheme.to_s
|
191
|
+
}
|
192
|
+
else
|
193
|
+
# New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
|
194
|
+
uri = URI(get_escaped_userinfo(host_str))
|
195
|
+
%w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
|
196
|
+
hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
|
197
|
+
hash
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end.compact
|
201
|
+
else
|
202
|
+
[{host: @host, port: @port, scheme: @scheme.to_s}]
|
203
|
+
end.each do |host|
|
204
|
+
host.merge!(user: @user, password: @password) if !host[:user] && @user
|
205
|
+
host.merge!(path: @path) if !host[:path] && @path
|
206
|
+
end
|
207
|
+
live_hosts = @check_connection ? hosts.select { |host| reachable_host?(host) } : hosts
|
208
|
+
{
|
209
|
+
hosts: live_hosts
|
210
|
+
}
|
211
|
+
end
|
212
|
+
|
213
|
+
def reachable_host?(host)
|
214
|
+
client = OpenSearch::Client.new(
|
215
|
+
host: ["#{host[:scheme]}://#{host[:host]}:#{host[:port]}"],
|
216
|
+
user: host[:user],
|
217
|
+
password: host[:password],
|
218
|
+
reload_connections: @reload_connections,
|
219
|
+
request_timeout: @request_timeout,
|
220
|
+
resurrect_after: @resurrect_after,
|
221
|
+
reload_on_failure: @reload_on_failure,
|
222
|
+
transport_options: { ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version } }
|
223
|
+
)
|
224
|
+
client.ping
|
225
|
+
rescue => e
|
226
|
+
log.warn "Failed to connect to #{host[:scheme]}://#{host[:host]}:#{host[:port]}: #{e.message}"
|
227
|
+
false
|
228
|
+
end
|
229
|
+
|
230
|
+
def emit_error_label_event(&block)
|
231
|
+
# If `emit_error_label_event` is specified as false, error event emittions are not occurred.
|
232
|
+
if emit_error_label_event
|
233
|
+
block.call
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
def start
|
238
|
+
super
|
239
|
+
|
240
|
+
timer_execute(:in_opensearch_timer, @interval, repeat: @repeat, &method(:run))
|
241
|
+
end
|
242
|
+
|
243
|
+
# We might be able to use
|
244
|
+
# Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
|
245
|
+
# [sec,nsec] where as we want something we can call `strftime` on...
|
246
|
+
def create_time_parser
|
247
|
+
if @timestamp_key_format
|
248
|
+
begin
|
249
|
+
# Strptime doesn't support all formats, but for those it does it's
|
250
|
+
# blazingly fast.
|
251
|
+
strptime = Strptime.new(@timestamp_key_format)
|
252
|
+
Proc.new { |value|
|
253
|
+
value = convert_numeric_time_into_string(value, @timestamp_key_format) if value.is_a?(Numeric)
|
254
|
+
strptime.exec(value).to_time
|
255
|
+
}
|
256
|
+
rescue
|
257
|
+
# Can happen if Strptime doesn't recognize the format; or
|
258
|
+
# if strptime couldn't be required (because it's not installed -- it's
|
259
|
+
# ruby 2 only)
|
260
|
+
Proc.new { |value|
|
261
|
+
value = convert_numeric_time_into_string(value, @timestamp_key_format) if value.is_a?(Numeric)
|
262
|
+
DateTime.strptime(value, @timestamp_key_format).to_time
|
263
|
+
}
|
264
|
+
end
|
265
|
+
else
|
266
|
+
Proc.new { |value|
|
267
|
+
value = convert_numeric_time_into_string(value) if value.is_a?(Numeric)
|
268
|
+
DateTime.parse(value).to_time
|
269
|
+
}
|
270
|
+
end
|
271
|
+
end
|
272
|
+
|
273
|
+
def convert_numeric_time_into_string(numeric_time, timestamp_key_format = "%Y-%m-%dT%H:%M:%S.%N%z")
|
274
|
+
numeric_time_parser = Fluent::NumericTimeParser.new(:float)
|
275
|
+
Time.at(numeric_time_parser.parse(numeric_time).to_r).strftime(timestamp_key_format)
|
276
|
+
end
|
277
|
+
|
278
|
+
def parse_time(value, event_time, tag)
|
279
|
+
@timestamp_parser.call(value)
|
280
|
+
rescue => e
|
281
|
+
emit_error_label_event do
|
282
|
+
router.emit_error_event(@timestamp_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @timestamp_key_format, 'value' => value}, e)
|
283
|
+
end
|
284
|
+
return Time.at(event_time).to_time
|
285
|
+
end
|
286
|
+
|
287
|
+
def client(host = nil)
|
288
|
+
# check here to see if we already have a client connection for the given host
|
289
|
+
connection_options = get_connection_options(host)
|
290
|
+
|
291
|
+
@_os = nil unless is_existing_connection(connection_options[:hosts])
|
292
|
+
|
293
|
+
@_os ||= begin
|
294
|
+
@current_config = connection_options[:hosts].clone
|
295
|
+
adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
|
296
|
+
local_reload_connections = @reload_connections
|
297
|
+
if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
|
298
|
+
local_reload_connections = @reload_after
|
299
|
+
end
|
300
|
+
|
301
|
+
headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
|
302
|
+
|
303
|
+
transport = OpenSearch::Transport::Transport::HTTP::Faraday.new(
|
304
|
+
connection_options.merge(
|
305
|
+
options: {
|
306
|
+
reload_connections: local_reload_connections,
|
307
|
+
reload_on_failure: @reload_on_failure,
|
308
|
+
resurrect_after: @resurrect_after,
|
309
|
+
logger: @transport_logger,
|
310
|
+
transport_options: {
|
311
|
+
headers: headers,
|
312
|
+
request: { timeout: @request_timeout },
|
313
|
+
ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
|
314
|
+
},
|
315
|
+
http: {
|
316
|
+
user: @user,
|
317
|
+
password: @password
|
318
|
+
},
|
319
|
+
sniffer_class: @sniffer_class,
|
320
|
+
}), &adapter_conf)
|
321
|
+
OpenSearch::Client.new transport: transport
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
def is_existing_connection(host)
|
326
|
+
# check if the host provided match the current connection
|
327
|
+
return false if @_os.nil?
|
328
|
+
return false if @current_config.nil?
|
329
|
+
return false if host.length != @current_config.length
|
330
|
+
|
331
|
+
for i in 0...host.length
|
332
|
+
if !host[i][:host].eql? @current_config[i][:host] || host[i][:port] != @current_config[i][:port]
|
333
|
+
return false
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
return true
|
338
|
+
end
|
339
|
+
|
340
|
+
def update_retry_state(error=nil)
|
341
|
+
if error
|
342
|
+
unless @retry
|
343
|
+
@retry = retry_state(@retry_randomize)
|
344
|
+
end
|
345
|
+
@retry.step
|
346
|
+
if error.message.include?('EOFError (EOFError)')
|
347
|
+
log.error("Restart plugin #{error.message}")
|
348
|
+
exit(1)
|
349
|
+
end
|
350
|
+
#Raise error if the retry limit has been reached
|
351
|
+
raise "Hit limit for retries. retry_times: #{@retry.steps}, error: #{error.message}" if @retry.limit?
|
352
|
+
#Retry if the limit hasn't been reached
|
353
|
+
log.warn("failed to connect or search.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, error: error.message)
|
354
|
+
sleep(@retry.next_time - Time.now)
|
355
|
+
else
|
356
|
+
unless @retry.nil?
|
357
|
+
log.info("retry succeeded.")
|
358
|
+
@retry = nil
|
359
|
+
end
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
def run
|
364
|
+
return run_slice if @num_slices <= 1
|
365
|
+
|
366
|
+
log.warn("Large slice number is specified:(#{@num_slices}). Consider reducing num_slices") if @num_slices > 8
|
367
|
+
|
368
|
+
@num_slices.times.map do |slice_id|
|
369
|
+
thread_create(:"in_opensearch_thread_#{slice_id}") do
|
370
|
+
run_slice(slice_id)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
rescue Faraday::ConnectionFailed, OpenSearch::Transport::Transport::Error => error
|
374
|
+
update_retry_state(error)
|
375
|
+
retry
|
376
|
+
end
|
377
|
+
|
378
|
+
def run_slice(slice_id=nil)
|
379
|
+
slice_query = @base_query
|
380
|
+
slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @num_slices}) unless slice_id.nil?
|
381
|
+
result = client.search(@options.merge(:body => Yajl.dump(slice_query) ))
|
382
|
+
es = Fluent::MultiEventStream.new
|
383
|
+
|
384
|
+
result["hits"]["hits"].each {|hit| process_events(hit, es)}
|
385
|
+
has_hits = result['hits']['hits'].any?
|
386
|
+
scroll_id = result['_scroll_id']
|
387
|
+
|
388
|
+
while has_hits && scroll_id
|
389
|
+
result = process_next_scroll_request(es, scroll_id)
|
390
|
+
has_hits = result['has_hits']
|
391
|
+
scroll_id = result['_scroll_id']
|
392
|
+
end
|
393
|
+
|
394
|
+
router.emit_stream(@tag, es)
|
395
|
+
clear_scroll(scroll_id)
|
396
|
+
update_retry_state
|
397
|
+
end
|
398
|
+
|
399
|
+
def clear_scroll(scroll_id)
|
400
|
+
client.clear_scroll(scroll_id: scroll_id) if scroll_id
|
401
|
+
rescue => e
|
402
|
+
# ignore & log any clear_scroll errors
|
403
|
+
log.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
|
404
|
+
end
|
405
|
+
|
406
|
+
def process_scroll_request(scroll_id)
|
407
|
+
client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
|
408
|
+
end
|
409
|
+
|
410
|
+
def process_next_scroll_request(es, scroll_id)
|
411
|
+
result = process_scroll_request(scroll_id)
|
412
|
+
result['hits']['hits'].each { |hit| process_events(hit, es) }
|
413
|
+
{'has_hits' => result['hits']['hits'].any?, '_scroll_id' => result['_scroll_id']}
|
414
|
+
end
|
415
|
+
|
416
|
+
def process_events(hit, es)
|
417
|
+
event = hit["_source"]
|
418
|
+
time = Fluent::Engine.now
|
419
|
+
if @parse_timestamp
|
420
|
+
if event.has_key?(TIMESTAMP_FIELD)
|
421
|
+
rts = event[TIMESTAMP_FIELD]
|
422
|
+
time = parse_time(rts, time, @tag)
|
423
|
+
end
|
424
|
+
end
|
425
|
+
if @docinfo
|
426
|
+
docinfo_target = event[@docinfo_target] || {}
|
427
|
+
|
428
|
+
unless docinfo_target.is_a?(Hash)
|
429
|
+
raise UnrecoverableError, "incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :type => docinfo_target.class, :event => event
|
430
|
+
end
|
431
|
+
|
432
|
+
@docinfo_fields.each do |field|
|
433
|
+
docinfo_target[field] = hit[field]
|
434
|
+
end
|
435
|
+
|
436
|
+
event[@docinfo_target] = docinfo_target
|
437
|
+
end
|
438
|
+
es.add(time, event)
|
439
|
+
end
|
440
|
+
end
|
441
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed to Uken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Uken Inc. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
require 'oj'
|
28
|
+
|
29
|
+
module Fluent::Plugin
|
30
|
+
module Serializer
|
31
|
+
|
32
|
+
class Oj
|
33
|
+
include OpenSearch::Transport::Transport::Serializer::Base
|
34
|
+
|
35
|
+
# De-serialize a Hash from JSON string
|
36
|
+
#
|
37
|
+
def load(string, options={})
|
38
|
+
::Oj.load(string, options)
|
39
|
+
end
|
40
|
+
|
41
|
+
# Serialize a Hash to JSON string
|
42
|
+
#
|
43
|
+
def dump(object, options={})
|
44
|
+
::Oj.dump(object, options)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed to Uken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Uken Inc. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
module Fluent
|
28
|
+
module Plugin
|
29
|
+
module OpenSearchConstants
|
30
|
+
BODY_DELIMITER = "\n".freeze
|
31
|
+
UPDATE_OP = "update".freeze
|
32
|
+
UPSERT_OP = "upsert".freeze
|
33
|
+
CREATE_OP = "create".freeze
|
34
|
+
INDEX_OP = "index".freeze
|
35
|
+
ID_FIELD = "_id".freeze
|
36
|
+
TIMESTAMP_FIELD = "@timestamp".freeze
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed to Uken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Uken Inc. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
require 'fluent/error'
|
28
|
+
|
29
|
+
class Fluent::Plugin::OpenSearchError
|
30
|
+
class RetryableOperationExhaustedFailure < Fluent::UnrecoverableError; end
|
31
|
+
end
|