test-input-opensearch 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.coveralls.yml +1 -0
- data/.editorconfig +9 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +29 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +9 -0
- data/.github/workflows/coverage.yaml +22 -0
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/.gitignore +18 -0
- data/CONTRIBUTING.md +24 -0
- data/Gemfile +10 -0
- data/History.md +67 -0
- data/LICENSE.txt +201 -0
- data/README.OpenSearchGenID.md +116 -0
- data/README.OpenSearchInput.md +396 -0
- data/README.Troubleshooting.md +482 -0
- data/README.md +1622 -0
- data/Rakefile +37 -0
- data/fluent-plugin-opensearch.gemspec +39 -0
- data/gemfiles/Gemfile.elasticsearch.v6 +12 -0
- data/lib/fluent/log-ext.rb +64 -0
- data/lib/fluent/plugin/filter_opensearch_genid.rb +103 -0
- data/lib/fluent/plugin/in_opensearch.rb +441 -0
- data/lib/fluent/plugin/oj_serializer.rb +48 -0
- data/lib/fluent/plugin/opensearch_constants.rb +39 -0
- data/lib/fluent/plugin/opensearch_error.rb +31 -0
- data/lib/fluent/plugin/opensearch_error_handler.rb +182 -0
- data/lib/fluent/plugin/opensearch_fallback_selector.rb +36 -0
- data/lib/fluent/plugin/opensearch_index_template.rb +155 -0
- data/lib/fluent/plugin/opensearch_simple_sniffer.rb +36 -0
- data/lib/fluent/plugin/opensearch_tls.rb +96 -0
- data/lib/fluent/plugin/out_opensearch.rb +1158 -0
- data/lib/fluent/plugin/out_opensearch_data_stream.rb +231 -0
- data/test/helper.rb +60 -0
- data/test/plugin/datastream_template.json +4 -0
- data/test/plugin/test_alias_template.json +9 -0
- data/test/plugin/test_filter_opensearch_genid.rb +241 -0
- data/test/plugin/test_in_opensearch.rb +500 -0
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_oj_serializer.rb +45 -0
- data/test/plugin/test_opensearch_error_handler.rb +770 -0
- data/test/plugin/test_opensearch_fallback_selector.rb +100 -0
- data/test/plugin/test_opensearch_tls.rb +171 -0
- data/test/plugin/test_out_opensearch.rb +3980 -0
- data/test/plugin/test_out_opensearch_data_stream.rb +746 -0
- data/test/plugin/test_template.json +23 -0
- data/test/test_log-ext.rb +61 -0
- metadata +291 -0
data/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed to Uken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Uken Inc. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
require 'bundler/gem_tasks'
|
28
|
+
require 'rake/testtask'
|
29
|
+
|
30
|
+
Rake::TestTask.new(:test) do |test|
|
31
|
+
test.libs << 'test'
|
32
|
+
test.pattern = 'test/**/test_*.rb'
|
33
|
+
test.verbose = true
|
34
|
+
test.warning = false
|
35
|
+
end
|
36
|
+
|
37
|
+
task :default => :test
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = 'test-input-opensearch'
|
6
|
+
s.version = '0.0.3'
|
7
|
+
s.authors = ['imcotop']
|
8
|
+
s.email = ['imcotop@icloud.com']
|
9
|
+
s.description = %q{Opensearch output plugin for Fluent event collector}
|
10
|
+
s.summary = s.description
|
11
|
+
s.homepage = 'https://github.com/fluent/fluent-plugin-opensearch'
|
12
|
+
s.license = 'Apache-2.0'
|
13
|
+
|
14
|
+
s.files = `git ls-files`.split($/)
|
15
|
+
s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
16
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
17
|
+
s.require_paths = ['lib']
|
18
|
+
|
19
|
+
if s.respond_to?(:metadata)
|
20
|
+
s.metadata["changelog_uri"] = "https://github.com/fluent/fluent-plugin-opensearch/blob/master/History.md"
|
21
|
+
end
|
22
|
+
|
23
|
+
s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
|
24
|
+
|
25
|
+
s.add_runtime_dependency 'fluentd', '>= 0.14.22'
|
26
|
+
s.add_runtime_dependency 'opensearch-ruby', '>= 3.0.1'
|
27
|
+
s.add_runtime_dependency "aws-sdk-core", "~> 3"
|
28
|
+
s.add_runtime_dependency 'excon', '>= 0'
|
29
|
+
s.add_runtime_dependency 'faraday', '>= 2.0.0'
|
30
|
+
s.add_runtime_dependency 'faraday-excon', '>= 2.0.0'
|
31
|
+
s.add_runtime_dependency "faraday_middleware-aws-sigv4", "~> 1.0.1"
|
32
|
+
|
33
|
+
s.add_development_dependency 'rake', '>= 0'
|
34
|
+
s.add_development_dependency 'webrick', '~> 1.7.0'
|
35
|
+
s.add_development_dependency 'webmock', '~> 3.18.1'
|
36
|
+
s.add_development_dependency 'test-unit', '~> 3.3.0'
|
37
|
+
s.add_development_dependency 'minitest', '~> 5.8'
|
38
|
+
s.add_development_dependency 'flexmock', '~> 2.0'
|
39
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
|
4
|
+
gemspec :path => "../"
|
5
|
+
|
6
|
+
gem 'simplecov', require: false
|
7
|
+
gem 'coveralls', ">= 0.8.0", require: false
|
8
|
+
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
|
+
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
|
+
gem "elasticsearch", "~> 6.8.1"
|
11
|
+
gem "elasticsearch-xpack"
|
12
|
+
gem "oj"
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright OpenSearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed toUken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Elasticsearch B.V. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
require 'fluent/log'
|
28
|
+
# For opensearch-ruby v1.0.0 or later which is based on elasticsearch-ruby v7.14 master tree
|
29
|
+
# logger for Elasticsearch::Loggable required the following methods:
|
30
|
+
#
|
31
|
+
# * debug?
|
32
|
+
# * info?
|
33
|
+
# * warn?
|
34
|
+
# * error?
|
35
|
+
# * fatal?
|
36
|
+
|
37
|
+
module Fluent
|
38
|
+
class Log
|
39
|
+
# OpenSearch::Loggable does not request trace? method.
|
40
|
+
# def trace?
|
41
|
+
# @level <= LEVEL_TRACE
|
42
|
+
# end
|
43
|
+
|
44
|
+
def debug?
|
45
|
+
@level <= LEVEL_DEBUG
|
46
|
+
end
|
47
|
+
|
48
|
+
def info?
|
49
|
+
@level <= LEVEL_INFO
|
50
|
+
end
|
51
|
+
|
52
|
+
def warn?
|
53
|
+
@level <= LEVEL_WARN
|
54
|
+
end
|
55
|
+
|
56
|
+
def error?
|
57
|
+
@level <= LEVEL_ERROR
|
58
|
+
end
|
59
|
+
|
60
|
+
def fatal?
|
61
|
+
@level <= LEVEL_FATAL
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
2
|
+
#
|
3
|
+
# The fluent-plugin-opensearch Contributors require contributions made to
|
4
|
+
# this file be licensed under the Apache-2.0 license or a
|
5
|
+
# compatible open source license.
|
6
|
+
#
|
7
|
+
# Modifications Copyright fluent-plugin-opensearch Contributors. See
|
8
|
+
# GitHub history for details.
|
9
|
+
#
|
10
|
+
# Licensed to Uken Inc. under one or more contributor
|
11
|
+
# license agreements. See the NOTICE file distributed with
|
12
|
+
# this work for additional information regarding copyright
|
13
|
+
# ownership. Uken Inc. licenses this file to you under
|
14
|
+
# the Apache License, Version 2.0 (the "License"); you may
|
15
|
+
# not use this file except in compliance with the License.
|
16
|
+
# You may obtain a copy of the License at
|
17
|
+
#
|
18
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
19
|
+
#
|
20
|
+
# Unless required by applicable law or agreed to in writing,
|
21
|
+
# software distributed under the License is distributed on an
|
22
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
23
|
+
# KIND, either express or implied. See the License for the
|
24
|
+
# specific language governing permissions and limitations
|
25
|
+
# under the License.
|
26
|
+
|
27
|
+
require 'securerandom'
|
28
|
+
require 'base64'
|
29
|
+
require 'fluent/plugin/filter'
|
30
|
+
|
31
|
+
module Fluent::Plugin
|
32
|
+
class OpenSearchGenidFilter < Filter
|
33
|
+
Fluent::Plugin.register_filter('opensearch_genid', self)
|
34
|
+
|
35
|
+
config_param :hash_id_key, :string, :default => '_hash'
|
36
|
+
config_param :include_tag_in_seed, :bool, :default => false
|
37
|
+
config_param :include_time_in_seed, :bool, :default => false
|
38
|
+
config_param :use_record_as_seed, :bool, :default => false
|
39
|
+
config_param :use_entire_record, :bool, :default => false
|
40
|
+
config_param :record_keys, :array, :default => []
|
41
|
+
config_param :separator, :string, :default => '_'
|
42
|
+
config_param :hash_type, :enum, list: [:md5, :sha1, :sha256, :sha512], :default => :sha1
|
43
|
+
|
44
|
+
def initialize
|
45
|
+
super
|
46
|
+
end
|
47
|
+
|
48
|
+
def configure(conf)
|
49
|
+
super
|
50
|
+
|
51
|
+
if !@use_entire_record
|
52
|
+
if @record_keys.empty? && @use_record_as_seed
|
53
|
+
raise Fluent::ConfigError, "When using record as hash seed, users must specify `record_keys`."
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
if @use_record_as_seed
|
58
|
+
class << self
|
59
|
+
alias_method :filter, :filter_seed_as_record
|
60
|
+
end
|
61
|
+
else
|
62
|
+
class << self
|
63
|
+
alias_method :filter, :filter_simple
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def filter(tag, time, record)
|
69
|
+
# for safety.
|
70
|
+
end
|
71
|
+
|
72
|
+
def filter_simple(tag, time, record)
|
73
|
+
record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
|
74
|
+
record
|
75
|
+
end
|
76
|
+
|
77
|
+
def filter_seed_as_record(tag, time, record)
|
78
|
+
seed = ""
|
79
|
+
seed += tag + separator if @include_tag_in_seed
|
80
|
+
seed += time.to_s + separator if @include_time_in_seed
|
81
|
+
if @use_entire_record
|
82
|
+
record.each {|k,v| seed += "|#{k}|#{v}"}
|
83
|
+
else
|
84
|
+
seed += record_keys.map {|k| record[k]}.join(separator)
|
85
|
+
end
|
86
|
+
record[@hash_id_key] = Base64.strict_encode64(encode_hash(@hash_type, seed))
|
87
|
+
record
|
88
|
+
end
|
89
|
+
|
90
|
+
def encode_hash(type, seed)
|
91
|
+
case type
|
92
|
+
when :md5
|
93
|
+
Digest::MD5.digest(seed)
|
94
|
+
when :sha1
|
95
|
+
Digest::SHA1.digest(seed)
|
96
|
+
when :sha256
|
97
|
+
Digest::SHA256.digest(seed)
|
98
|
+
when :sha512
|
99
|
+
Digest::SHA512.digest(seed)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|