fluent-plugin-input-opensearch 1.1.9

data/lib/fluent/plugin/oj_serializer.rb

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'oj'
+
+module Fluent::Plugin
+  module Serializer
+
+    class Oj
+      include OpenSearch::Transport::Transport::Serializer::Base
+
+      # De-serialize a Hash from JSON string
+      #
+      def load(string, options={})
+        ::Oj.load(string, options)
+      end
+
+      # Serialize a Hash to JSON string
+      #
+      def dump(object, options={})
+        ::Oj.dump(object, options)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/opensearch_constants.rb

@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Fluent
+  module Plugin
+    module OpenSearchConstants
+      BODY_DELIMITER = "\n".freeze
+      UPDATE_OP = "update".freeze
+      UPSERT_OP = "upsert".freeze
+      CREATE_OP = "create".freeze
+      INDEX_OP = "index".freeze
+      ID_FIELD = "_id".freeze
+      TIMESTAMP_FIELD = "@timestamp".freeze
+    end
+  end
+end

data/lib/fluent/plugin/opensearch_error.rb

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'fluent/error'
+
+class Fluent::Plugin::OpenSearchError
+  class RetryableOperationExhaustedFailure < Fluent::UnrecoverableError; end
+end

data/lib/fluent/plugin/opensearch_error_handler.rb

@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'fluent/event'
+require 'fluent/error'
+require_relative 'opensearch_constants'
+
+class Fluent::Plugin::OpenSearchErrorHandler
+  include Fluent::Plugin::OpenSearchConstants
+
+  attr_accessor :bulk_message_count
+  class OpenSearchVersionMismatch < Fluent::UnrecoverableError; end
+  class OpenSearchSubmitMismatch < Fluent::UnrecoverableError; end
+  class OpenSearchRequestAbortError < Fluent::UnrecoverableError; end
+  class OpenSearchError < StandardError; end
+
+  def initialize(plugin)
+    @plugin = plugin
+  end
+
+  def unrecoverable_error_types
+    @plugin.unrecoverable_error_types
+  end
+
+  def unrecoverable_error?(type)
+    unrecoverable_error_types.include?(type)
+  end
+
+  def unrecoverable_record_types
+    @plugin.unrecoverable_record_types
+  end
+
+  def unrecoverable_record_error?(type)
+    unrecoverable_record_types.include?(type)
+  end
+
+  def log_os_400_reason(&block)
+    if @plugin.log_os_400_reason
+      block.call
+    else
+      @plugin.log.on_debug(&block)
+    end
+  end
+
+  def emit_error_label_event?
+    !!@plugin.emit_error_label_event
+  end
+
+  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+    items = response['items']
+    if items.nil? || !items.is_a?(Array)
+      raise OpenSearchVersionMismatch, "The response format was unrecognized: #{response}"
+    end
+    if bulk_message_count != items.length
+      raise OpenSearchSubmitMismatch, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
+    end
+    retry_stream = Fluent::MultiEventStream.new
+    stats = Hash.new(0)
+    meta = {}
+    header = {}
+    affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
+    chunk.msgpack_each do |time, rawrecord|
+      bulk_message = ''
+      next unless rawrecord.is_a? Hash
+      begin
+        # we need a deep copy for process_message to alter
+        processrecord = Marshal.load(Marshal.dump(rawrecord))
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
+        next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
+      rescue => e
+        @plugin.log.debug("Exception in error handler during deep copy: #{e}")
+        stats[:bad_chunk_record] += 1
+        next
+      end
+      item = items.shift
+      if item.is_a?(Hash) && item.has_key?(@plugin.write_operation)
+        write_operation = @plugin.write_operation
+      elsif INDEX_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(CREATE_OP)
+        write_operation = CREATE_OP
+      elsif UPSERT_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(UPDATE_OP)
+        write_operation = UPDATE_OP
+      elsif item.nil?
+        stats[:errors_nil_resp] += 1
+        next
+      else
+        # When we don't have an expected ops field, something changed in the API
+        # expected return values.
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      if item[write_operation].has_key?('status')
+        status = item[write_operation]['status']
+      else
+        # When we don't have a status field, something changed in the API
+        # expected return values.
+        stats[:errors_bad_resp] += 1
+        next
+      end
+      case
+      when [200, 201].include?(status)
+        stats[:successes] += 1
+      when CREATE_OP == write_operation && 409 == status
+        stats[:duplicates] += 1
+      when 400 == status
+        stats[:bad_argument] += 1
+        reason = ""
+        log_os_400_reason do
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+            reason = " [error type]: #{item[write_operation]['error']['type']}"
+          end
+          if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('reason')
+            reason += " [reason]: \'#{item[write_operation]['error']['reason']}\'"
+          end
+        end
+        if emit_error_label_event?
+          @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("400 - Rejected by OpenSearch#{reason}"))
+        end
+      else
+        if item[write_operation]['error'].is_a?(String)
+          reason = item[write_operation]['error']
+          stats[:errors_block_resp] += 1
+          if emit_error_label_event?
+            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{reason}"))
+          end
+          next
+        elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
+          type = item[write_operation]['error']['type']
+          stats[type] += 1
+          if unrecoverable_error?(type)
+            raise OpenSearchRequestAbortError, "Rejected OpenSearch due to #{type}"
+          end
+          if unrecoverable_record_error?(type)
+            if emit_error_label_event?
+              @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - #{type}: #{reason}"))
+            end
+            next
+          else
+            retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
+          end
+        else
+          # When we don't have a type field, something changed in the API
+          # expected return values.
+          stats[:errors_bad_resp] += 1
+          if emit_error_label_event?
+            @plugin.router.emit_error_event(tag, time, rawrecord, OpenSearchError.new("#{status} - No error type provided in the response"))
+          end
+          next
+        end
+        stats[type] += 1
+      end
+    end
+    @plugin.log.on_debug do
+      msg = ["Indexed (op = #{@plugin.write_operation})"]
+      stats.each_pair { |key, value| msg << "#{value} #{key}" }
+      @plugin.log.debug msg.join(', ')
+    end
+    raise Fluent::Plugin::OpenSearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
+  end
+end

data/lib/fluent/plugin/opensearch_fallback_selector.rb

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+require 'opensearch/transport/transport/connections/selector'
+
+class Fluent::Plugin::OpenSearchFallbackSelector
+  include OpenSearch::Transport::Transport::Connections::Selector::Base
+
+  def select(options={})
+    connections.first
+  end
+end

data/lib/fluent/plugin/opensearch_index_template.rb

@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'fluent/error'
+require_relative './opensearch_error'
+
+module Fluent::OpenSearchIndexTemplate
+  def get_template(template_file)
+    if !File.exist?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    JSON.parse(file_contents)
+  end
+
+  def get_custom_template(template_file, customize_template)
+    if !File.exist?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    customize_template.each do |key, value|
+      file_contents = file_contents.gsub(key,value.downcase)
+    end
+    JSON.parse(file_contents)
+  end
+
+  def template_exists?(name, host = nil)
+    if @use_legacy_template
+      client(host).indices.get_template(:name => name)
+    else
+      client(host).indices.get_index_template(:name => name)
+    end
+    return true
+  rescue OpenSearch::Transport::Transport::Errors::NotFound
+    return false
+  end
+
+  def host_unreachable_exceptions
+    client.transport.transport.host_unreachable_exceptions
+  end
+
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_transport_exceptions = true)
+    return unless block_given?
+    retries = 0
+    transport_errors = OpenSearch::Transport::Transport::Errors.constants.map{ |c| OpenSearch::Transport::Transport::Errors.const_get c } if catch_transport_exceptions
+    begin
+      yield
+    rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
+      @_es = nil
+      @_es_info = nil
+      if retries < max_retries
+        retries += 1
+        wait_seconds = 2**retries
+        sleep wait_seconds
+        log.warn "Could not communicate to OpenSearch, resetting connection and trying again. #{e.message}"
+        log.warn "Remaining retry: #{max_retries - retries}. Retry to communicate after #{wait_seconds} second(s)."
+        retry
+      end
+      message = "Could not communicate to OpenSearch after #{retries} retries. #{e.message}"
+      log.warn message
+      raise Fluent::Plugin::OpenSearchError::RetryableOperationExhaustedFailure,
+            message if fail_on_retry_exceed
+    end
+  end
+
+  def template_put(name, template, host = nil)
+    if @use_legacy_template
+      client(host).indices.put_template(:name => name, :body => template)
+    else
+      client(host).indices.put_index_template(:name => name, :body => template)
+    end
+  end
+
+  def indexcreation(index_name, host = nil)
+    client(host).indices.create(:index => index_name)
+  rescue OpenSearch::Transport::Transport::Error => e
+    if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
+      log.debug("Index #{index_name} already exists")
+    else
+      log.error("Error while index creation - #{index_name}", error: e)
+    end
+  end
+
+  def template_install(name, template_file, overwrite, host = nil, target_index = nil, index_separator = '-')
+    if overwrite
+      template_put(name,
+                   get_template(template_file), host)
+
+      log.debug("Template '#{name}' overwritten with #{template_file}.")
+      return
+    end
+    if !template_exists?(name, host)
+      template_put(name,
+                   get_template(template_file), host)
+      log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
+    else
+      log.debug("Template '#{name}' configured and already installed.")
+    end
+  end
+
+  def template_custom_install(template_name, template_file, overwrite, customize_template, host, target_index, index_separator)
+    custom_template = get_custom_template(template_file, customize_template)
+
+    if overwrite
+      template_put(template_name, custom_template, host)
+      log.info("Template '#{template_name}' overwritten with #{template_file}.")
+    else
+      if !template_exists?(template_name, host)
+        template_put(template_name, custom_template, host)
+        log.info("Template configured, but no template installed. Installed '#{template_name}' from #{template_file}.")
+      else
+        log.debug("Template '#{template_name}' configured and already installed.")
+      end
+    end
+  end
+
+  def templates_hash_install(templates, overwrite)
+    templates.each do |key, value|
+      template_install(key, value, overwrite)
+    end
+  end
+
+  def rollover_alias_payload(rollover_alias)
+    {
+      'aliases' => {
+        rollover_alias => {
+          'is_write_index' =>  true
+        }
+      }
+    }
+  end
+end

data/lib/fluent/plugin/opensearch_simple_sniffer.rb

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'opensearch'
+
+class Fluent::Plugin::OpenSearchSimpleSniffer < OpenSearch::Transport::Transport::Sniffer
+
+  def hosts
+    @transport.logger.debug "In Fluent::Plugin::OpenSearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
+    @transport.hosts
+  end
+
+end

data/lib/fluent/plugin/opensearch_tls.rb

@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'openssl'
+require 'fluent/configurable'
+require 'fluent/config/error'
+
+module Fluent::Plugin
+  module OpenSearchTLS
+    SUPPORTED_TLS_VERSIONS = if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                               [:TLSv1, :TLSv1_1, :TLSv1_2, :TLSv1_3].freeze
+                             else
+                               [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
+                             end
+
+    DEFAULT_VERSION = :TLSv1_2
+    METHODS_MAP = begin
+                    # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
+                    map = {
+                      TLSv1: OpenSSL::SSL::TLS1_VERSION,
+                      TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
+                      TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
+                    }
+                    map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                    USE_TLS_MINMAX_VERSION = true
+                    map.freeze
+                  rescue NameError
+                    map = {
+                      SSLv23: :SSLv23,
+                      TLSv1: :TLSv1,
+                      TLSv1_1: :TLSv1_1,
+                      TLSv1_2: :TLSv1_2,
+                    }
+                    USE_TLS_MINMAX_VERSION = false
+                  end
+    private_constant :METHODS_MAP
+
+    module OpenSearchTLSParams
+      include Fluent::Configurable
+
+      config_param :ssl_version, :enum, list: Fluent::Plugin::OpenSearchTLS::SUPPORTED_TLS_VERSIONS, default: Fluent::Plugin::OpenSearchTLS::DEFAULT_VERSION
+      config_param :ssl_min_version, :enum, list: Fluent::Plugin::OpenSearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
+      config_param :ssl_max_version, :enum, list: Fluent::Plugin::OpenSearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
+    end
+
+    def self.included(mod)
+      mod.include OpenSearchTLSParams
+    end
+
+    def set_tls_minmax_version_config(ssl_version, ssl_max_version, ssl_min_version)
+      if USE_TLS_MINMAX_VERSION
+        case
+        when ssl_min_version.nil? && ssl_max_version.nil?
+          ssl_min_version = METHODS_MAP[:TLSv1_2]
+          ssl_max_version = METHODS_MAP[:TLSv1_3]
+        when ssl_min_version && ssl_max_version.nil?
+          raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
+        when ssl_min_version.nil? && ssl_max_version
+          raise Fluent::ConfigError, "When you set 'ssl_max_version', must set 'ssl_min_version' together."
+        else
+          ssl_min_version = METHODS_MAP[ssl_min_version]
+          ssl_max_version = METHODS_MAP[ssl_max_version]
+        end
+
+        {max_version: ssl_max_version, min_version: ssl_min_version}
+      else
+        log.warn "'ssl_min_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_min_version.nil?
+        log.warn "'ssl_max_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_max_version.nil?
+        {version: ssl_version}
+      end
+    end
+  end
+end