test-input-opensearch-1 0.0.3

data/Rakefile

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.warning = false
+end
+
+task :default => :test

data/fluent-plugin-opensearch.gemspec

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = 'test-input-opensearch-1'
+  s.version       = '0.0.3'
+  s.authors       = ['imcotop']
+  s.email         = ['imcotop@icloud.com']
+  s.description   = %q{Opensearch output plugin for Fluent event collector}
+  s.summary       = s.description
+  s.homepage      = 'https://github.com/fluent/fluent-plugin-opensearch'
+  s.license       = 'Apache-2.0'
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+
+  if s.respond_to?(:metadata)
+    s.metadata["changelog_uri"] = "https://github.com/fluent/fluent-plugin-opensearch/blob/master/History.md"
+  end
+
+  s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
+
+  s.add_runtime_dependency 'fluentd', '>= 0.14.22'
+  s.add_runtime_dependency 'opensearch-ruby', '>= 3.0.1'
+  s.add_runtime_dependency "aws-sdk-core", "~> 3"
+  s.add_runtime_dependency 'excon', '>= 0'
+  s.add_runtime_dependency 'faraday', '>= 2.0.0'
+  s.add_runtime_dependency 'faraday-excon', '>= 2.0.0'
+  s.add_runtime_dependency "faraday_middleware-aws-sigv4", "~> 1.0.1"
+
+  s.add_development_dependency 'rake', '>= 0'
+  s.add_development_dependency 'webrick', '~> 1.7.0'
+  s.add_development_dependency 'webmock', '~> 3.18.1'
+  s.add_development_dependency 'test-unit', '~> 3.3.0'
+  s.add_development_dependency 'minitest', '~> 5.8'
+  s.add_development_dependency 'flexmock', '~> 2.0'
+end

data/gemfiles/Gemfile.elasticsearch.v6

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
+gemspec :path => "../"
+
+gem 'simplecov', require: false
+gem 'coveralls', ">= 0.8.0", require: false
+gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
+gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "elasticsearch", "~> 6.8.1"
+gem "elasticsearch-xpack"
+gem "oj"

data/lib/fluent/log-ext.rb

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+# Licensed toUken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'fluent/log'
+# For opensearch-ruby v1.0.0 or later which is based on elasticsearch-ruby v7.14 master tree
+# logger for Elasticsearch::Loggable required the following methods:
+#
+# * debug?
+# * info?
+# * warn?
+# * error?
+# * fatal?
+
+module Fluent
+  class Log
+    # OpenSearch::Loggable does not request trace? method.
+    # def trace?
+    #   @level <= LEVEL_TRACE
+    # end
+
+    def debug?
+      @level <= LEVEL_DEBUG
+    end
+
+    def info?
+      @level <= LEVEL_INFO
+    end
+
+    def warn?
+      @level <= LEVEL_WARN
+    end
+
+    def error?
+      @level <= LEVEL_ERROR
+    end
+
+    def fatal?
+      @level <= LEVEL_FATAL
+    end
+  end
+end

data/lib/fluent/plugin/filter_opensearch_genid.rb

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'securerandom'
+require 'base64'
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+  class OpenSearchGenidFilter < Filter
+    Fluent::Plugin.register_filter('opensearch_genid', self)
+
+    config_param :hash_id_key, :string, :default => '_hash'
+    config_param :include_tag_in_seed, :bool, :default => false
+    config_param :include_time_in_seed, :bool, :default => false
+    config_param :use_record_as_seed, :bool, :default => false
+    config_param :use_entire_record, :bool, :default => false
+    config_param :record_keys, :array, :default => []
+    config_param :separator, :string, :default => '_'
+    config_param :hash_type, :enum, list: [:md5, :sha1, :sha256, :sha512], :default => :sha1
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+
+      if !@use_entire_record
+        if @record_keys.empty? && @use_record_as_seed
+          raise Fluent::ConfigError, "When using record as hash seed, users must specify `record_keys`."
+        end
+      end
+
+      if @use_record_as_seed
+        class << self
+          alias_method :filter, :filter_seed_as_record
+        end
+      else
+        class << self
+          alias_method :filter, :filter_simple
+        end
+      end
+    end
+
+    def filter(tag, time, record)
+      # for safety.
+    end
+
+    def filter_simple(tag, time, record)
+      record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
+      record
+    end
+
+    def filter_seed_as_record(tag, time, record)
+      seed = ""
+      seed += tag + separator if @include_tag_in_seed
+      seed += time.to_s + separator if @include_time_in_seed
+      if @use_entire_record
+        record.each {|k,v| seed += "|#{k}|#{v}"}
+      else
+        seed += record_keys.map {|k| record[k]}.join(separator)
+      end
+      record[@hash_id_key] = Base64.strict_encode64(encode_hash(@hash_type, seed))
+      record
+    end
+
+    def encode_hash(type, seed)
+      case type
+      when :md5
+        Digest::MD5.digest(seed)
+      when :sha1
+        Digest::SHA1.digest(seed)
+      when :sha256
+        Digest::SHA256.digest(seed)
+      when :sha512
+        Digest::SHA512.digest(seed)
+      end
+    end
+  end
+end