logstash-integration-elastic_enterprise_search 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fe83963cbb7d00dffdf8b52106d2db2e34f8b5389c891a7a1db7f939657c6ed2
+  data.tar.gz: 21be61a90719d99c958a9d4eed07a67a4d5ea22d6fbcf5cb8531a3af5e036c01
+SHA512:
+  metadata.gz: f4fece8fe3bec522750956526422d44da6b0ea1b2bcf42d01ca992f5835f67ab5e173e9543442da453f97dc68c78cf87fab8a7370925d7696e68069eeb20f16d
+  data.tar.gz: 88ba3212c15e0f84a2023472a80b4acbb81663db628d6ef209cba3295ec4310a78b06e21c1c3a31a552ea08e249faf04a6fc8efa5f3f52998dddd1557a8b2eb6

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+## 2.0.0
+
+- Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
+  previous AppSearch Output plugin codebase; 
+  independent changelogs for previous versions can be found:
+   - [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
+

data/CONTRIBUTORS

@@ -0,0 +1,12 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Joao Duarte - jsvduarte@gmail.com
+* Peter Kim - petedogg@gmail.com
+* Andrea Selva (andsel)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,5 @@
+# logstash-integration-elastic_enterprise_search
+Elastic Enterprise Search integration for Logstash, including AppSearch and WorkplaceSearch output plugins
+
+# Dependencies
+ * elastic-app-search

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,100 @@
+# logstash-integration-elastic_enterprise_search
+
+
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-example.svg)](https://travis-ci.com/logstash-plugins/logstash-output-example)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically build documentation for this plugin. We provide a template file, index.asciidoc, where you can add documentation. The contents of this file will be converted into html and then placed with other plugin documentation in a [central location](http://www.elastic.co/guide/en/logstash/current/).
+
+- For formatting config examples, you can use the asciidoc `[source,json]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
+>>>>>>> 418bfff... Rought output plugin creation

data/lib/logstash/outputs/elastic_app_search.rb

@@ -0,0 +1,117 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "elastic-app-search"
+
+class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
+  config_name "elastic_app_search"
+
+  config :engine, :validate => :string, :required => true
+  config :host, :validate => :string
+  config :url, :validate => :string
+  config :api_key, :validate => :password, :required => true
+  config :timestamp_destination, :validate => :string
+  config :document_id, :validate => :string
+  config :path, :validate => :string, :default => "/api/as/v1/"
+
+  ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
+
+  public
+  def register
+    if @host.nil? && @url.nil?
+      raise ::LogStash::ConfigurationError.new("Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
+    elsif @host && @url
+      raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
+    elsif @host && path_is_set?  # because path has a default value we need extra work to if the user set it
+      raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
+    elsif @host
+      @client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
+    elsif @url
+      @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
+    end
+    check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
+  rescue => e
+    if e.message =~ /401/
+      raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. Error: 401. Please check your credentials")
+    elsif e.message =~ /404/
+      raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. Error: 404. Please check if host '#{@host}' is correct and you've created an engine with name '#{@engine}'")
+    else
+      raise ::LogStash::ConfigurationError.new("Failed to connect to App Search. #{e.message}")
+    end
+  end
+
+  public
+  def multi_receive(events)
+    # because App Search has a limit of 100 documents per bulk
+    events.each_slice(100) do |events|
+      batch = format_batch(events)
+      if @logger.trace?
+        @logger.trace("Sending bulk to AppSearch", :size => batch.size, :data => batch.inspect)
+      end
+      index(batch)
+    end
+  end
+
+  private
+  def format_batch(events)
+    docs_for_engine = {}
+    events.each do |event|
+      doc = event.to_hash
+      # we need to remove default fields that start with "@"
+      # since Elastic App Search doesn't accept them
+      if @timestamp_destination
+        doc[@timestamp_destination] = doc.delete("@timestamp")
+      else # delete it
+        doc.delete("@timestamp")
+      end
+      if @document_id
+        doc["id"] = event.sprintf(@document_id)
+      end
+      doc.delete("@version")
+      resolved_engine = event.sprintf(@engine)
+      unless docs_for_engine[resolved_engine]
+        if @logger.debug?
+          @logger.debug("Creating new engine segment in batch to send", :resolved_engine => resolved_engine)
+        end
+        docs_for_engine[resolved_engine] = []
+      end
+      docs_for_engine[resolved_engine] << doc
+    end
+    docs_for_engine
+  end
+
+  def index(batch)
+    batch.each do |resolved_engine, documents|
+      begin
+        if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
+          raise "Cannot resolve engine field name #{@engine} from event"
+        end
+        response = @client.index_documents(resolved_engine, documents)
+        report(documents, response)
+      rescue => e
+        @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
+                      :resolved_engine => resolved_engine)
+        sleep(1)
+        retry
+      end
+    end
+  end
+
+  def report(documents, response)
+    documents.each_with_index do |document, i|
+      errors = response[i]["errors"]
+      if errors.empty?
+        @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
+      else
+        @logger.warn("Document failed to index. Dropping..", :document => document, :errors => errors.to_a)
+      end
+    end
+  end
+
+  def check_connection!
+    @client.get_engine(@engine)
+  end
+
+  def path_is_set?
+    original_params.key?("path")
+  end
+end

data/logstash-integration-elastic_enterprise_search.gemspec

@@ -0,0 +1,31 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-integration-elastic_enterprise_search'
+  s.version         = '2.0.0'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "Integration with Elastic Enterprise Search - output plugins"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
+                      "using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program."
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths   = ['lib', 'vendor/jar-dependencies']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+    # Special flag to let us know this is actually a logstash plugin
+  s.metadata = {
+      "logstash_plugin"     => "true",
+      "logstash_group"      => "integration",
+      "integration_plugins" => "logstash-output-elastic_app_search"
+  }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/integration/outputs/index_spec.rb

@@ -0,0 +1,180 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elastic_app_search"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+
+describe "indexing against running AppSearch", :integration => true do
+
+  let(:engine_name) do
+    (0...10).map { ('a'..'z').to_a[rand(26)] }.join
+  end
+
+  let(:config) do
+    {
+      "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+      "engine" => engine_name,
+      "url" => "http://appsearch:3002"
+    }
+  end
+
+  subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
+
+  before(:each) do
+    create_engine(engine_name, "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+  end
+
+  private
+  def create_engine(engine_name, host, api_key)
+    url = host + "/api/as/v1/engines"
+    resp = Faraday.post(url, "{\"name\": \"#{engine_name}\"}",
+      "Content-Type" => "application/json",
+      "Authorization" => "Bearer " + api_key)
+    expect(resp.status).to eq(200)
+  end
+
+  describe "search and private keys are configured" do
+    let(:api_key_settings) do
+      {
+        :private => ENV['APPSEARCH_PRIVATE_KEY'],
+        :search => ENV['APPSEARCH_SEARCH_KEY']
+      }
+    end
+
+    it "setup apikeys" do
+      expect(api_key_settings[:private]).to start_with("private-")
+      expect(api_key_settings[:search]).to start_with("search-")
+    end
+  end
+
+  describe "register" do
+    let(:config) do
+      {
+        "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+        "engine" => "%{engine_name_field}",
+        "url" => "http://appsearch:3002"
+      }
+    end
+
+    context "when engine is defined in sprintf format" do
+      it "does not raise an error" do
+        expect { subject.register }.to_not raise_error
+      end
+    end
+  end
+
+  describe "indexing" do
+
+    before do
+      app_search_output.register
+    end
+
+    describe "single event" do
+      let(:event) { LogStash::Event.new("message" => "an event to index") }
+
+      it "should be indexed" do
+        app_search_output.multi_receive([event])
+
+        results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+          attempt_response = execute_search_call(engine_name)
+          expect(attempt_response.status).to eq(200)
+          parsed_resp = JSON.parse(attempt_response.body)
+          expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
+          parsed_resp["results"]
+        end
+        expect(results.first.dig("message", "raw")).to eq "an event to index"
+      end
+
+      context "using sprintf-ed engine" do
+        let(:config) do
+          {
+            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "engine" => "%{engine_name_field}",
+            "url" => "http://appsearch:3002"
+          }
+        end
+
+        let(:event) { LogStash::Event.new("message" => "an event to index", "engine_name_field" => engine_name) }
+
+        it "should be indexed" do
+          app_search_output.multi_receive([event])
+
+          results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+            attempt_response = execute_search_call(engine_name)
+            expect(attempt_response.status).to eq(200)
+            parsed_resp = JSON.parse(attempt_response.body)
+            expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
+            parsed_resp["results"]
+          end
+          expect(results.first.dig("message", "raw")).to eq "an event to index"
+        end
+      end
+    end
+
+    private
+    def execute_search_call(engine_name)
+      url = config["url"] + "/api/as/v1/engines/#{engine_name}/search"
+      resp = Faraday.post(url, '{"query": "event"}',
+            "Content-Type" => "application/json",
+            "Authorization" => "Bearer " + config["api_key"])
+    end
+
+    describe "multiple events" do
+      context "single static engine" do
+        let(:events) { generate_events(200) } #2 times the slice size used to batch
+
+        it "all should be indexed" do
+          app_search_output.multi_receive(events)
+
+          expect_indexed(engine_name, 200)
+        end
+      end
+
+      context "multiple sprintf engines" do
+        let(:config) do
+          {
+            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "engine" => "%{engine_name_field}",
+            "url" => "http://appsearch:3002"
+          }
+        end
+
+        it "all should be indexed" do
+         create_engine('testengin1', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+         create_engine('testengin2', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+         events = generate_events(100, 'testengin1')
+         events += generate_events(100, 'testengin2')
+         events.shuffle!
+
+         app_search_output.multi_receive(events)
+
+         expect_indexed('testengin1', 100)
+         expect_indexed('testengin2', 100)
+        end
+      end
+    end
+
+    private
+    def expect_indexed(engine_name, expected_docs_count)
+      results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+        attempt_response = execute_search_call(engine_name)
+        expect(attempt_response.status).to eq(200)
+        parsed_resp = JSON.parse(attempt_response.body)
+        expect(parsed_resp.dig("meta", "page", "total_results")).to eq(expected_docs_count)
+        parsed_resp["results"]
+      end
+      expect(results.first.dig("message", "raw")).to start_with("an event to index")
+    end
+
+    def generate_events(num_events, engine_name = nil)
+      (1..num_events).map do |i|
+        if engine_name
+          LogStash::Event.new("message" => "an event to index #{i}", "engine_name_field" => engine_name)
+        else
+          LogStash::Event.new("message" => "an event to index #{i}")
+        end
+      end
+    end
+  end
+end

data/spec/unit/outputs/appsearch_spec.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elastic_app_search"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::ElasticAppSearch do
+  let(:sample_event) { LogStash::Event.new }
+  let(:host) { "test-host" }
+  let(:api_key) { "my_key" }
+  let(:engine) { "test-engine" }
+  subject { described_class.new(config) }
+
+  describe "#register" do
+    before(:each) do
+      allow(subject).to receive(:check_connection!)
+    end
+    context "when host is configured" do
+      let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine } }
+      it "does not raise an error" do
+        expect { subject.register }.to_not raise_error
+      end
+    end
+    context "when host and path is configured" do
+      let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine, "path" => "/v1" } }
+      it "raises an error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+    context "when host and url is configured" do
+      let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine, "url" => "http://localhost:9300" } }
+      it "raises an error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+    context "when neither host nor url is configured" do
+      let(:config) { { "api_key" => api_key, "engine" => engine } }
+      it "raises an error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+    context "when engine is in sprintf format" do
+      let(:config) { { "host" => host, "api_key" => api_key, "engine" => "%{type}" } }
+      it "connection is not checked" do
+        expect { subject.register }.to_not raise_error
+        expect(subject).not_to receive(:check_connection!)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: logstash-integration-elastic_enterprise_search
+version: !ruby/object:Gem::Version
+  version: 2.0.0
+platform: ruby
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.8.0
+  name: elastic-app-search
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program.
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/outputs/elastic_app_search.rb
+- logstash-integration-elastic_enterprise_search.gemspec
+- spec/integration/outputs/index_spec.rb
+- spec/unit/outputs/appsearch_spec.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: integration
+  integration_plugins: logstash-output-elastic_app_search
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- vendor/jar-dependencies
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: Integration with Elastic Enterprise Search - output plugins
+test_files:
+- spec/integration/outputs/index_spec.rb
+- spec/unit/outputs/appsearch_spec.rb