fluent-plugin-opensearch 1.0.0

data/lib/fluent/plugin/out_opensearch_data_stream.rb

@@ -0,0 +1,214 @@
+
+require_relative 'out_opensearch'
+
+module Fluent::Plugin
+  class OpenSearchOutputDataStream < OpenSearchOutput
+
+    Fluent::Plugin.register_output('opensearch_data_stream', self)
+
+    helpers :event_emitter
+
+    config_param :data_stream_name, :string
+    config_param :data_stream_template_name, :string, :default => nil
+    # OpenSearch 1.0 or later always support new style of index template.
+    config_set_default :use_legacy_template, false
+
+    INVALID_START_CHRACTERS = ["-", "_", "+", "."]
+    INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
+
+    def configure(conf)
+      super
+
+      @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
+
+      # ref. https://opensearch.org/docs/latest/opensearch/data-streams/
+      unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
+        validate_data_stream_parameters
+      else
+        @use_placeholder = true
+        @data_stream_names = []
+      end
+
+      @client = client
+      unless @use_placeholder
+        begin
+          @data_stream_names = [@data_stream_name]
+          create_index_template(@data_stream_name, @data_stream_template_name, @host)
+          create_data_stream(@data_stream_name)
+        rescue => e
+          raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
+        end
+      end
+    end
+
+    def validate_data_stream_parameters
+      {"data_stream_name" => @data_stream_name,
+       "data_stream_template_name"=> @data_stream_template_name}.each do |parameter, value|
+        unless valid_data_stream_parameters?(value)
+          unless start_with_valid_characters?(value)
+            if not_dots?(value)
+              raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
+            else
+              raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
+            end
+          end
+          unless valid_characters?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
+          end
+          unless lowercase_only?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
+          end
+          if value.bytes.size > 255
+            raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
+          end
+        end
+      end
+    end
+
+    def create_index_template(datastream_name, template_name, host)
+      return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
+      body = {
+        "index_patterns" => ["#{datastream_name}*"],
+        "data_stream" => {},
+      }
+      params = {
+        name: template_name,
+        body: body
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        @client.indices.put_index_template(params)
+      end
+    end
+
+    def data_stream_exist?(datastream_name)
+      params = {
+        name: datastream_name
+      }
+      begin
+        # TODO: Use X-Pack equivalent performing DataStream operation method on the following line
+        response = @client.perform_request('GET', "/_data_stream/#{datastream_name}", {}, params)
+        return (not response.is_a?(OpenSearch::Transport::Transport::Errors::NotFound))
+      rescue OpenSearch::Transport::Transport::Errors::NotFound => e
+        log.info "Specified data stream does not exist. Will be created: <#{e}>"
+        return false
+      end
+    end
+
+    def create_data_stream(datastream_name)
+      return if data_stream_exist?(datastream_name)
+      params = {
+        name: datastream_name
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        # TODO: Use X-Pack equivalent performing DataStream operation method on the following line
+        @client.perform_request('PUT', "/_data_stream/#{datastream_name}", {}, params)
+      end
+    end
+
+    def template_exists?(name, host = nil)
+      if @use_legacy_template
+        client(host).indices.get_template(:name => name)
+      else
+        client(host).indices.get_index_template(:name => name)
+      end
+      return true
+    rescue OpenSearch::Transport::Transport::Errors::NotFound
+      return false
+    end
+
+    def valid_data_stream_parameters?(data_stream_parameter)
+      lowercase_only?(data_stream_parameter) and
+        valid_characters?(data_stream_parameter) and
+        start_with_valid_characters?(data_stream_parameter) and
+        not_dots?(data_stream_parameter) and
+        data_stream_parameter.bytes.size <= 255
+    end
+
+    def lowercase_only?(data_stream_parameter)
+      data_stream_parameter.downcase == data_stream_parameter
+    end
+
+    def valid_characters?(data_stream_parameter)
+      not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
+    end
+
+    def start_with_valid_characters?(data_stream_parameter)
+      not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
+    end
+
+    def not_dots?(data_stream_parameter)
+      not (data_stream_parameter == "." or data_stream_parameter == "..")
+    end
+
+    def client_library_version
+      OpenSearch::VERSION
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def write(chunk)
+      data_stream_name = @data_stream_name
+      data_stream_template_name = @data_stream_template_name
+      host = @host
+      if @use_placeholder
+        data_stream_name = extract_placeholders(@data_stream_name, chunk)
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        unless @data_stream_names.include?(data_stream_name)
+          begin
+            create_index_template(data_stream_name, data_stream_template_name, host)
+            create_data_stream(data_stream_name)
+            @data_stream_names << data_stream_name
+          rescue => e
+            raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
+          end
+        end
+      end
+
+      bulk_message = ""
+      headers = {
+        CREATE_OP => {}
+      }
+      tag = chunk.metadata.tag
+      chunk.msgpack_each do |time, record|
+        next unless record.is_a? Hash
+
+        begin
+          record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      end
+
+      params = {
+        index: data_stream_name,
+        body: bulk_message
+      }
+      begin
+        response = @client.bulk(params)
+        if response['errors']
+          log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
+        end
+      rescue => e
+        raise RecoverableRequestFailure, "could not push logs to OpenSearch cluster (#{data_stream_name}): #{e.message}"
+      end
+    end
+
+    def append_record_to_messages(op, meta, header, record, msgs)
+      header[CREATE_OP] = meta
+      msgs << @dump_proc.call(header) << BODY_DELIMITER
+      msgs << @dump_proc.call(record) << BODY_DELIMITER
+      msgs
+    end
+
+    def retry_stream_retryable?
+      @buffer.storable?
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,61 @@
+# coding: utf-8
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'simplecov'
+require 'simplecov'
+require 'simplecov-lcov'
+
+SimpleCov::Formatter::LcovFormatter.config do |config|
+  config.report_with_single_file = true
+  config.single_report_path = 'coverage/lcov.info'
+end
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
+  SimpleCov::Formatter::HTMLFormatter,
+  SimpleCov::Formatter::LcovFormatter
+])
+
+SimpleCov.start do
+  add_filter do |src|
+    !(src.filename =~ /^#{SimpleCov.root}\/lib/)
+  end
+end
+
+# needs to be after simplecov but before test/unit, because fluentd sets default
+# encoding to ASCII-8BIT, but coverall might load git data which could contain a
+# UTF-8 character
+at_exit do
+  Encoding.default_internal = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_internal)
+  Encoding.default_external = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_external)
+end
+
+require 'test/unit'
+require 'fluent/test'
+require 'minitest/pride'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!

data/test/plugin/test_alias_template.json

@@ -0,0 +1,9 @@
+{
+  "order": 5,
+  "template": "--index_prefix-----appid---*",
+  "settings": {},
+  "mappings": {},
+  "aliases": {
+    "--appid---alias": {}
+  }
+}

data/test/plugin/test_filter_opensearch_genid.rb

@@ -0,0 +1,241 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The fluent-plugin-opensearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright fluent-plugin-opensearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Uken Inc. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Uken Inc. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../helper'
+require 'date'
+require 'fluent/test/helpers'
+require 'json'
+require 'fluent/test/driver/filter'
+require 'flexmock/test_unit'
+require 'fluent/plugin/filter_opensearch_genid'
+
+class OpenSearchGenidFilterTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='')
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::OpenSearchGenidFilter).configure(conf)
+  end
+
+  test "invalid configuration" do
+    assert_raise(Fluent::ConfigError) do
+      create_driver("use_record_as_seed true")
+    end
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal '_hash', d.instance.hash_id_key
+  end
+
+  data("default" => {"hash_id_key" => "_hash"},
+       "custom_key" => {"hash_id_key" => "_edited"},
+      )
+  def test_filter(data)
+    d = create_driver("hash_id_key #{data["hash_id_key"]}")
+    flexmock(SecureRandom).should_receive(:uuid)
+      .and_return("13a0c028-bf7c-4ae2-ad03-ec09a40006df")
+    time = event_time("2017-10-15 15:00:23.34567890 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, sample_record)
+    end
+    assert_equal(Base64.strict_encode64(SecureRandom.uuid),
+                 d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+  end
+
+  class UseRecordAsSeedTest < self
+    data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
+         "sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
+         "sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
+         "sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
+        )
+    def test_simple(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
+         "sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
+         "sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
+         "sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
+    def test_record_with_tag(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
+         "sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
+         "sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
+         "sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
+    def test_record_with_time(data)
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
+         "sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
+         "sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
+         "sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        record_keys age,parent_id,routing_id,custom_key
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
+
+  class UseEntireRecordAsSeedTest < self
+    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
+         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
+         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
+         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    def test_record
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
+         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
+         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
+         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    def test_record_with_tag
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
+         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
+         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
+         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    def test_record_with_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+
+    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
+         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
+         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
+         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    def test_record_with_tag_and_time
+      hash_type, expected = data
+      d = create_driver(%[
+        use_record_as_seed true
+        use_entire_record true
+        hash_type #{hash_type}
+        include_tag_in_seed true
+        include_time_in_seed true
+      ])
+      time = event_time("2017-10-15 15:00:23.34567890 UTC")
+      d.run(default_tag: 'test.fluentd') do
+        d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
+      end
+      assert_equal(expected,
+                   d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
+    end
+  end
+end