logstash-input-opensearch 1.0.0

data/lib/logstash/inputs/opensearch.rb

@@ -0,0 +1,432 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require "logstash/util/safe_uri"
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require "base64"
+
+require "opensearch"
+require "opensearch/transport/transport/http/manticore"
+require_relative "opensearch/patches/_opensearch_transport_http_manticore"
+require_relative "opensearch/patches/_opensearch_transport_connections_selector"
+
+# .Compatibility Note
+# [NOTE]
+# ================================================================================
+# Starting with OpenSearch 5.3, there's an {ref}modules-http.html[HTTP setting]
+# called `http.content_type.required`. If this option is set to `true`, and you
+# are using Logstash 2.4 through 5.2, you need to update the OpenSearch input
+# plugin to version 4.0.2 or higher.
+# 
+# ================================================================================
+# 
+# Read from an OpenSearch cluster, based on search query results.
+# This is useful for replaying test logs, reindexing, etc.
+# It also supports periodically scheduling lookup enrichments
+# using a cron syntax (see `schedule` setting).
+#
+# Example:
+# [source,ruby]
+#     input {
+#       # Read all documents from OpenSearch matching the given query
+#       opensearch {
+#         hosts => "localhost"
+#         query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }'
+#       }
+#     }
+#
+# This would create an OpenSearch query with the following format:
+# [source,json]
+#     curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{
+#       "query": {
+#         "match": {
+#           "statuscode": 200
+#         }
+#       },
+#       "sort": [ "_doc" ]
+#     }'
+#
+# ==== Scheduling
+#
+# Input from this plugin can be scheduled to run periodically according to a specific
+# schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler].
+# The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ).
+#
+# Examples:
+#
+# |==========================================================
+# | `* 5 * 1-3 *`               | will execute every minute of 5am every day of January through March.
+# | `0 * * * *`                 | will execute on the 0th minute of every hour every day.
+# | `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day.
+# |==========================================================
+#
+#
+# Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
+#
+#
+class LogStash::Inputs::OpenSearch < LogStash::Inputs::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
+  config_name "opensearch"
+
+  # List of opensearch hosts to use for querying.
+  # Each host can be either IP, HOST, IP:port or HOST:port.
+  # Port defaults to 9200
+  config :hosts, :validate => :array
+
+  # The index or alias to search.
+  config :index, :validate => :string, :default => "logstash-*"
+
+  # The query to be executed. Read the OpenSearch query DSL documentation
+  # for more info
+  # https://opensearch.org/docs/latest/opensearch/query-dsl/index/
+  config :query, :validate => :string, :default => '{ "sort": [ "_doc" ] }'
+
+  # This allows you to set the maximum number of hits returned per scroll.
+  config :size, :validate => :number, :default => 1000
+
+  # This parameter controls the keepalive time in seconds of the scrolling
+  # request and initiates the scrolling process. The timeout applies per
+  # round trip (i.e. between the previous scroll request, to the next).
+  config :scroll, :validate => :string, :default => "1m"
+
+  # This parameter controls the number of parallel slices to be consumed simultaneously
+  # by this pipeline input.
+  config :slices, :validate => :number
+
+  # If set, include OpenSearch document information such as index, type, and
+  # the id in the event.
+  #
+  # It might be important to note, with regards to metadata, that if you're
+  # ingesting documents with the intent to re-index them (or just update them)
+  # that the `action` option in the opensearch output wants to know how to
+  # handle those things. It can be dynamically assigned with a field
+  # added to the metadata.
+  #
+  # Example
+  # [source, ruby]
+  #     input {
+  #       opensearch {
+  #         hosts => "es.production.mysite.org"
+  #         index => "mydata-2018.09.*"
+  #         query => "*"
+  #         size => 500
+  #         scroll => "5m"
+  #         docinfo => true
+  #       }
+  #     }
+  #     output {
+  #       opensearch {
+  #         index => "copy-of-production.%{[@metadata][_index]}"
+  #         document_type => "%{[@metadata][_type]}"
+  #         document_id => "%{[@metadata][_id]}"
+  #       }
+  #     }
+  #
+  config :docinfo, :validate => :boolean, :default => false
+
+  # Where to move the OpenSearch document information.
+  # default: [@metadata][input][opensearch] in ECS mode, @metadata field otherwise
+  config :docinfo_target, :validate=> :field_reference
+
+  # List of document metadata to move to the `docinfo_target` field.
+  config :docinfo_fields, :validate => :array, :default => ['_index', '_type', '_id']
+
+  # Basic Auth - username
+  config :user, :validate => :string
+
+  # Basic Auth - password
+  config :password, :validate => :password
+
+  # Connection Timeout, in Seconds
+  config :connect_timeout_seconds, :validate => :positive_whole_number, :default => 10
+
+  # Request Timeout, in Seconds
+  config :request_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Socket Timeout, in Seconds
+  config :socket_timeout_seconds, :validate => :positive_whole_number, :default => 60
+
+  # Set the address of a forward HTTP proxy.
+  config :proxy, :validate => :uri_or_empty
+
+  # SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary 
+  config :ca_file, :validate => :path
+
+  # Schedule of when to periodically run statement, in Cron format
+  # for example: "* * * * *" (execute query every minute, on the minute)
+  #
+  # There is no schedule by default. If no schedule is given, then the statement is run
+  # exactly once.
+  config :schedule, :validate => :string
+
+  # If set, the _source of each hit will be added nested under the target instead of at the top-level
+  config :target, :validate => :field_reference
+
+  def initialize(params={})
+    super(params)
+
+    if docinfo_target.nil?
+      @docinfo_target = ecs_select[disabled: '@metadata', v1: '[@metadata][input][opensearch]']
+    end
+  end
+
+  def register
+    require "rufus/scheduler"
+
+    @options = {
+      :index => @index,
+      :scroll => @scroll,
+      :size => @size
+    }
+    @base_query = LogStash::Json.load(@query)
+    if @slices
+      @base_query.include?('slice') && fail(LogStash::ConfigurationError, "OpenSearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
+      @slices < 1 && fail(LogStash::ConfigurationError, "OpenSearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
+    end
+
+    transport_options = {:headers => {}}
+    transport_options[:headers].merge!(setup_basic_auth(user, password))
+    transport_options[:headers].merge!({'user-agent' => prepare_user_agent()})
+    transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
+    transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
+    transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
+
+    hosts = setup_hosts
+    ssl_options = setup_ssl
+
+    @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
+
+    transport_options[:proxy] = @proxy.to_s if @proxy && !@proxy.eql?('')
+
+    @client = OpenSearch::Client.new(
+      :hosts => hosts,
+      :transport_options => transport_options,
+      :transport_class => ::OpenSearch::Transport::Transport::HTTP::Manticore,
+      :ssl => ssl_options
+    )
+    test_connection!
+    @client
+  end
+
+
+  def run(output_queue)
+    if @schedule
+      @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
+      @scheduler.cron @schedule do
+        do_run(output_queue)
+      end
+
+      @scheduler.join
+    else
+      do_run(output_queue)
+    end
+  end
+
+  def stop
+    @scheduler.stop if @scheduler
+  end
+
+  private
+
+  def do_run(output_queue)
+    # if configured to run a single slice, don't bother spinning up threads
+    return do_run_slice(output_queue) if @slices.nil? || @slices <= 1
+
+    logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
+
+    @slices.times.map do |slice_id|
+      Thread.new do
+        LogStash::Util::set_thread_name("#{@id}_slice_#{slice_id}")
+        do_run_slice(output_queue, slice_id)
+      end
+    end.map(&:join)
+  end
+
+  def do_run_slice(output_queue, slice_id=nil)
+    slice_query = @base_query
+    slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
+
+    slice_options = @options.merge(:body => LogStash::Json.dump(slice_query) )
+
+    logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+
+    begin
+      r = search_request(slice_options)
+
+      r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+      logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+
+      has_hits = r['hits']['hits'].any?
+      scroll_id = r['_scroll_id']
+
+      while has_hits && scroll_id && !stop?
+        has_hits, scroll_id = process_next_scroll(output_queue, scroll_id)
+        logger.debug("Slice progress", slice_id: slice_id, slices: @slices) if logger.debug? && slice_id
+      end
+      logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+    ensure
+      clear_scroll(scroll_id)
+    end
+  end
+
+  ##
+  # @param output_queue [#<<]
+  # @param scroll_id [String]: a scroll id to resume
+  # @return [Array(Boolean,String)]: a tuple representing whether the response
+  #
+  def process_next_scroll(output_queue, scroll_id)
+    r = scroll_request(scroll_id)
+    r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+    [r['hits']['hits'].any?, r['_scroll_id']]
+  rescue => e
+    # this will typically be triggered by a scroll timeout
+    logger.error("Scroll request error, aborting scroll", message: e.message, exception: e.class)
+    # return no hits and original scroll_id so we can try to clear it
+    [false, scroll_id]
+  end
+
+  def push_hit(hit, output_queue)
+    event = targeted_event_factory.new_event hit['_source']
+    set_docinfo_fields(hit, event) if @docinfo
+    decorate(event)
+    output_queue << event
+  end
+
+  def set_docinfo_fields(hit, event)
+    # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
+    docinfo_target = event.get(@docinfo_target) || {}
+
+    unless docinfo_target.is_a?(Hash)
+      @logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata)
+
+      # TODO: (colin) I am not sure raising is a good strategy here?
+      raise Exception.new("OpenSearch input: incompatible event")
+    end
+
+    @docinfo_fields.each do |field|
+      docinfo_target[field] = hit[field]
+    end
+
+    event.set(@docinfo_target, docinfo_target)
+  end
+
+  def clear_scroll(scroll_id)
+    @client.clear_scroll(:body => { :scroll_id => scroll_id }) if scroll_id
+  rescue => e
+    # ignore & log any clear_scroll errors
+    logger.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
+  end
+
+  def scroll_request scroll_id
+    @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
+  end
+
+  def search_request(options)
+    @client.search(options)
+  end
+
+  def hosts_default?(hosts)
+    hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
+  end
+
+  def setup_ssl
+    return { :ssl  => true, :ca_file => @ca_file } if @ssl && @ca_file
+    return { :ssl  => true, :verify => false } if @ssl  # Setting verify as false if ca_file is not provided
+  end
+
+  def setup_hosts
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+    if @ssl
+      @hosts.map do |h|
+        host, port = h.split(":")
+        { :host => host, :scheme => 'https', :port => port }
+      end
+    else
+      @hosts
+    end
+  end
+
+  def setup_basic_auth(user, password)
+    return {} unless user && password && password.value
+
+    token = ::Base64.strict_encode64("#{user}:#{password.value}")
+    { 'Authorization' => "Basic #{token}" }
+  end
+
+  def prepare_user_agent
+      os_name = java.lang.System.getProperty('os.name')
+      os_version = java.lang.System.getProperty('os.version')
+      os_arch = java.lang.System.getProperty('os.arch')
+      jvm_vendor = java.lang.System.getProperty('java.vendor')
+      jvm_version = java.lang.System.getProperty('java.version')
+
+      plugin_version = Gem.loaded_specs["logstash-input-opensearch"].version
+      # example: logstash/7.14.1 (OS=Linux-5.4.0-84-generic-amd64; JVM=AdoptOpenJDK-11.0.11) logstash-input-opensearch/4.10.0
+      "logstash/#{LOGSTASH_VERSION} (OS=#{os_name}-#{os_version}-#{os_arch}; JVM=#{jvm_vendor}-#{jvm_version}) logstash-#{@plugin_type}-#{config_name}/#{plugin_version}"
+  end
+
+  # @private used by unit specs
+  attr_reader :client
+
+  def test_connection!
+    @client.ping
+  rescue OpenSearch::UnsupportedProductError
+    raise LogStash::ConfigurationError, "Could not connect to a compatible version of OpenSearch"
+  end
+
+  module URIOrEmptyValidator
+    ##
+    # @override to provide :uri_or_empty validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :uri_or_empty
+
+      value = deep_replace(value)
+      value = hash_or_array(value)
+
+      return true, value.first if value.size == 1 && value.first.empty?
+
+      return super(value, :uri)
+    end
+  end
+  extend(URIOrEmptyValidator)
+
+  module PositiveWholeNumberValidator
+    ##
+    # @override to provide :positive_whole_number validator
+    # @param value [Array<Object>]
+    # @param validator [nil,Array,Symbol]
+    # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+    # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+    def validate_value(value, validator)
+      return super unless validator == :positive_whole_number
+
+      is_number, coerced_number = super(value, :number)
+
+      return [true, coerced_number.to_i] if is_number && coerced_number.denominator == 1 && coerced_number > 0
+
+      return [false, "Expected positive whole number, got `#{value.inspect}`"]
+    end
+  end
+  extend(PositiveWholeNumberValidator)
+end

data/logstash-input-opensearch.gemspec

@@ -0,0 +1,52 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+signing_key_path = "private_key.pem"
+
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-input-opensearch'
+  s.version         = '1.0.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Reads query results from an OpenSearch cluster"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic", "OpenSearch Contributors"]
+  s.email           = 'opensearch@amazon.com'
+  s.homepage        = "https://opensearch.org/"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  if $PROGRAM_NAME.end_with?("gem") && ARGV == ["build", __FILE__] && File.exist?(signing_key_path)
+    s.signing_key = signing_key_path
+    s.cert_chain  = ['certs/opensearch-rubygems.pem']
+    s.signing_key = File.expand_path("private_key.pem") if $0 =~ /gem\z/
+  end
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
+  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
+
+  s.add_runtime_dependency 'tzinfo'
+  s.add_runtime_dependency 'tzinfo-data'
+  s.add_runtime_dependency 'rufus-scheduler'
+  s.add_runtime_dependency 'manticore', ">= 0.7.1"
+
+  s.add_runtime_dependency 'opensearch-ruby'
+
+  s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'faraday', "~> 1"
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'timecop'
+  s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'webrick'
+end

data/spec/fixtures/test_certs/ca.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD
+VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr
+8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr
+PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd
+OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe
+oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz
+YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC
+AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY
+MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB
+G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL
+DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL
+sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36
+NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz
+emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/ca.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
+QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
+90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
+99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
+i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
+m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABAoIBABmBC0P6Ebegljkk
+lO26GdbOKvbfqulDS3mN5QMyXkUMopea03YzMnKUJriE+2O33a1mUcuDPWnLpYPK
+BTiQieYHlulNtY0Bzf+R69igRq9+1WpZftGnzrlu7NVxkOokRqWJv3546ilV7QZ0
+f9ngmu+tiN7hEnlBC8m613VMuGGb3czwbCizEVZxlZX0Dk2GExbH7Yf3NNs/aOP/
+8x6CqgL+rhrtOQ80xwRrOlEF8oSSjXCzypa3nFv21YO3J2lVo4BoIwnHgOzyz46A
+b37gekqXXajIYQ0HAB+NDgVoCRFFJ7Xe16mgB3DpyUpUJzwiMedJkeQ0TprIownQ
++1mPe9ECgYEA/K4jc0trr3sk8KtcZjOYdpvwrhEqSSGEPeGfFujZaKOb8PZ8PX6j
+MbCTV12nEgm8FEhZQ3azxLnO17gbJ2A+Ksm/IIwnTWlqvvMZD5qTQ7L3qZuCtbWQ
++EGC/H1SDjhiwvjHcXP61/tYL/peApBSoj0L4kC+U/VaNyvicudKk08CgYEAr46J
+4VJBJfZ4ZaUBRy53+fy+mknOfaj2wo8MnD3u+/x4YWTapqvDOPN2nJVtKlIsxbS4
+qCO+fzUV17YHlsQmGULNbtFuXWJkP/RcLVbe8VYg/6tmk0dJwNAe90flagX2KJov
+8eDX129nNpuUqrNNWsfeLmPmH6vUzpKlga+1zfUCgYBrbUHHJ96dmbZn2AMNtIvy
+iXP3HXcj5msJwB3aKJ8eHMkU1kaWAnwxiQfrkfaQ9bCP0v6YbyQY1IJ7NlvdDs7/
+dAydMtkW0WW/zyztdGN92d3vrx0QUiRTV87vt/wl7ZUXnZt1wcB5CPRCWaiUYHWx
+YlDmHW6N1XdIk5DQF0OegwKBgEt7S8k3Zo9+A5IgegYy8p7njsQjy8a3qTFJ9DAR
+aPmrOc8WX/SdkVihRXRZwxAZOOrgoyyYAcYL+xI+T9EBESh3UoC9R2ibb2MYG7Ha
+0gyN7a4/8eCNHCbs1QOZRAhr+8TFVqv28pbMbWJLToZ+hVns6Zikl0MyzFLtNoAm
+HlMpAoGBAIOkqnwwuRKhWprL59sdcJfWY26os9nvuDV4LoKFNEFLJhj2AA2/3UlV
+v85gqNSxnMNlHLZC9l2HZ3mKv/mfx1aikmFvyhJAnk5u0f9KkexmCPLjQzS5q3ba
+yFuxK2DXwN4x46RgQPFlLjOTCX0BG6rkEu4JdonF8ETSjoCtGEU8
+-----END RSA PRIVATE KEY-----

data/spec/fixtures/test_certs/es.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL
+BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
+cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
+VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu
+1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T
+Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt
+CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ
+2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX
+LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ
+tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX
+MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs
+YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc
+7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW
+f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE
+Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+
+LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA
+qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX
+qi02i4q6meHGcw==
+-----END CERTIFICATE-----

data/spec/fixtures/test_certs/es.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5VV21nXpYzQJoQ
+buWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz36pcFw7UyF51
+/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/cMjNrUC7iP0dv
+fOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH/z07/mVKoBAa
+5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gRhQNM3zcKKsjE
+MomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABAoIBAQCm/VBDz41ImG7p
+yu3e6iMeFi7HW5SKdlRUS5dJbHT1uBWJAm/q8TbwvnUBVdsn9cKWY06QYDPQBjAy
+0LxRSIKivjyl+aIJDZbbEUXrmk/M0zT9rHtgSc2isM8ITH6IHw5q7lmNMPLYOu6T
+IMvfTDtADBOOTV/vF+/4NKf5GCUXVt1XTzLBFMK0p/ZoI7Fsw7fhH6FR12vk0xA4
+BEC4pwRbGfHo7P31ii0by8epkve93tF4IZuFmN92A84bN1z7Kc4TYaSbua2rgguz
+FzMyWpsTxr363HzCK1xOJb6JyJOiXbq4+j2oqtne3GIvyozJeiyKRgjLIMoe/LV7
+fPPc5wlhAoGBAOD3z0JH2eyR/1RHILFsWInH2nDbKHHuCjhFIL2XloeXsJkiJZ95
+BpdjExMZCqD44tPNRW/GgWKwoVwltm6zB0aq0aW/OfOzw6fhKt1W+go47L7Tpwap
+VQgy6BFXSueUKfQDlZEWV4E2gakf8vOl0/VRQExae/CeKf1suEedQaErAoGBAMWE
+LOmNDEU2NFqghfNBAFYyFJst3YnBmSmlL7W22+OsfSK/PhxnJbuNHxMgxpg9rieW
+tVyjuZRo/i7WLVm3uG+dK1RJ9t8Y6kpYkCRKpi9G8DBOj3PSulOybBr+fdRfW9mf
+8UmqOjOkrhxXPkchc9TY4EM7/1XeKvEidlIp0gvRAoGAAurz4zYvW2QhXaR2hhaT
+p2XSLXiKM8AUndo3rH3U0/lhrvrEZicZsMj2LF88xg20U27sIaD/eJo13Y4XqaPk
+ykPY6D9srv574SeIeMpx/8PxPiBcoDd+BNc0L1VkgVBoouORAwq5I9HjKKBjdEmI
+UDw3i0X5KYvDm6fXVAZ0HXUCgYBWc4To8KiXPqNpq2sVzrSkBaWJSmj2G7u7Q6b/
+RTs3is72v3gjHG6iiaE5URY7mnu4rjlRhAP9Vnsy6uHMrCJZEBTf/sPEYHZj9iGZ
+EOduOAF3U1tsmaaebbDtm8hdhSOBvITy9kQlSIZAt1r17Ulytz5pj0AySFzJUIkz
+a0SZkQKBgCWixtUxiK8PAdWhyS++90WJeJn8eqjuSAz+VMtFQFRRWDUbkiHvGMRu
+o/Hhk6zS46gSF2Evb1d26uUEenXnJlIp6YWzb0DLPrfy5P53kPA6YEvYq5MSAg3l
+DZOJUF+ko7cWXSZkeTIBH/jrGOdP4tTALZt6DNt+Gz7xwPO5tGgV
+-----END RSA PRIVATE KEY-----

data/spec/inputs/integration/opensearch_spec.rb

@@ -0,0 +1,83 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin"
+require "logstash/inputs/opensearch"
+require_relative "../../../spec/opensearch_helper"
+
+describe LogStash::Inputs::OpenSearch do
+
+  let(:config)   { { 'hosts' => [OpenSearchHelper.get_host_port],
+                     'index' => 'logs',
+                     'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
+  let(:plugin) { described_class.new(config) }
+  let(:event)  { LogStash::Event.new({}) }
+  let(:client_options) { Hash.new }
+
+  before(:each) do
+    @es = OpenSearchHelper.get_client(client_options)
+    # Delete all templates first.
+    # Clean OpenSearch of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    10.times do
+      OpenSearchHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
+    end
+    @es.indices.refresh
+  end
+
+  after(:each) do
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  shared_examples 'an opensearch index plugin' do
+    before(:each) do
+      plugin.register
+    end
+
+    it 'should retrieve json event from opensearch' do
+      queue = []
+      plugin.run(queue)
+      event = queue.pop
+      expect(event).to be_a(LogStash::Event)
+      expect(event.get("response")).to eql(404)
+    end
+  end
+
+  describe 'against an unsecured opensearch', :integration => true do
+    before(:each) do
+      plugin.register
+    end
+
+    it_behaves_like 'an opensearch index plugin'
+  end
+
+  describe 'against a secured opensearch', :secure_integration => true do
+    let(:user) { ENV['USER'] || 'admin' }
+    let(:password) { ENV['PASSWORD'] || 'admin' }
+
+    let(:client_options) { { :user => user, :password => password } }
+
+    let(:config) { super().merge('user' => user, 'password' => password, 'ssl' => true) }
+
+    it_behaves_like 'an opensearch index plugin'
+
+    context "incorrect auth credentials" do
+
+      let(:config) do
+        super().merge('user' => 'archer', 'password' => 'b0gus!')
+      end
+
+      let(:queue) { [] }
+
+      it "fails to run the plugin" do
+        expect { plugin.register }.to raise_error OpenSearch::Transport::Transport::Errors::Unauthorized
+      end
+    end
+
+  end
+end