logstash-integration-elastic_enterprise_search 2.1.2 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a26f0ede537038c1f3b7acb68be793d6b1a9ff6b1114e5635499914f8fdb3605
-  data.tar.gz: 67f3f5969a96e0421ad1f350af3aceec37e0745df0ff94af609645caf8e8ac41
+  metadata.gz: 6276fe38b651d58dcfa68a99d6d6de58f4b7a35d4e5cf640f996cbd7b681a7ad
+  data.tar.gz: 373697b1c10c3c51af094d0415e407033ea1d987d776f4c0e0a83146b998f2cf
 SHA512:
-  metadata.gz: 5779393bb76e0066720bce165fffcda3cdd4dfb4503ac9ee9819ccd380ab1f53c566321f258455a9f0dc0597c0863568a39b6ee61f6e77e7e0c2e60b6de1b564
-  data.tar.gz: 0e9bf91d6888da92db60965e835a9cfd1044441b9fe7a2e433d9c9b9efe795550165128056d488efb04bd64a714a5bb070908e977cf42190b266aa5161ff5fc5
+  metadata.gz: 898a93fb3d0516b4b6f2a0eb358a29809e581a56ed7c5ca3be21e95f778dd4e63b81e615dbe9bfb96a22cc1439d3f840f0e574a4b56de4e08de3a5eff36ccce9
+  data.tar.gz: 9f4edd07607d41ee54ec36141dd0b009ae8eb5c24364b05a200b419e2efdd800ed57694f10c150e56c1186ee54639fff31361a6c72e9ecff44fb431f251c82b9

data/CHANGELOG.md

@@ -1,19 +1,24 @@
+
+## 2.2.0
+ - Feature, switch the connection library to elastic-enterprise-search [#3](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/3)
+ - [DOC] Added required parameters to Workplace Search example snippet and describe little better what's expected in url parameter [#11](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/11)
+
 ## 2.1.2
-- [DOC] Fix typos in App Search and Workplace Search [#9](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
-- [DOC] Add links to Elastic App Search and Elastic Workplace Search solution pages [#10](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9) 
+ - [DOC] Fix typos in App Search and Workplace Search [#9](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
+ - [DOC] Add links to Elastic App Search and Elastic Workplace Search solution pages [#10](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9) 
 
 ## 2.1.1
-- [DOC] Added the integration attribute and include statement for the integration plugin header to finalize hooking up integration docs [#8](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/8)
-- [DOC] Added live link to output-elastic_workplace_search doc [#6](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/6)
+ - [DOC] Added the integration attribute and include statement for the integration plugin header to finalize hooking up integration docs [#8](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/8)
+ - [DOC] Added live link to output-elastic_workplace_search doc [#6](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/6)
 
 ## 2.1.0
-- Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
-- [DOC] Added the integration attribute and include statement for the integration plugin header to output-elastic_app_search doc [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
+ - Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
+ - [DOC] Added the integration attribute and include statement for the integration plugin header to output-elastic_app_search doc [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
 
 
 ## 2.0.0
-- Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
-  previous AppSearch Output plugin codebase; 
-  independent changelogs for previous versions can be found:
-   - [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
+ - Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
+   previous AppSearch Output plugin codebase; 
+   independent changelogs for previous versions can be found:
+    - [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
 

data/DEVELOPER.md

@@ -2,5 +2,6 @@
 Elastic Enterprise Search integration for Logstash, including App Search and Workplace Search output plugins
 
 # Dependencies
- * elastic-app-search
+ * elastic-enterprise-search
+ * elastic-app-search  
  * elastic-workplace-search

data/lib/logstash/outputs/elastic_app_search.rb

@@ -1,8 +1,12 @@
 # encoding: utf-8
 require "logstash/outputs/base"
 require "elastic-app-search"
+require "elastic-enterprise-search"
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
+  include LogStash::PluginMixins::DeprecationLoggerSupport
+
   config_name "elastic_app_search"
 
   # The name of the search engine you created in App Search, an information
@@ -49,16 +53,25 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
 
   public
   def register
+    @use_old_client = false
     if @host.nil? && @url.nil?
       raise ::LogStash::ConfigurationError.new("Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
     elsif @host && @url
-      raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
+      raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed ot Elastic Enterprise Search) or \"host\" (for SaaS).")
     elsif @host && path_is_set?  # because path has a default value we need extra work to if the user set it
       raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
     elsif @host
+      @deprecation_logger.deprecated("Deprecated service usage, the `host` setting will be removed when Swiftype AppSearch service is shutdown")
+      @use_old_client = true
       @client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
     elsif @url
-      @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
+      if path_is_set?
+        @deprecation_logger.deprecated("Deprecated service usage, the `path` setting will be removed when Swiftype AppSearch service is shutdown")
+        @use_old_client = true
+        @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
+      else
+        @client = Elastic::EnterpriseSearch::AppSearch::Client.new(:host => @url, :http_auth => @api_key.value, :external_url => @url)
+      end
     end
     check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
   rescue => e
@@ -117,11 +130,15 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
         if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
           raise "Cannot resolve engine field name #{@engine} from event"
         end
-        response = @client.index_documents(resolved_engine, documents)
+        if connected_to_swiftype?
+          response = @client.index_documents(resolved_engine, documents)
+        else
+          response = @client.index_documents(resolved_engine, {:documents => documents})
+        end
         report(documents, response)
       rescue => e
         @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
-                      :resolved_engine => resolved_engine)
+                      :resolved_engine => resolved_engine, :backtrace => e.backtrace)
         sleep(1)
         retry
       end
@@ -130,7 +147,11 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
 
   def report(documents, response)
     documents.each_with_index do |document, i|
-      errors = response[i]["errors"]
+      if connected_to_swiftype?
+        errors = response[i]["errors"]
+      else
+        errors = response.body[i]["errors"]
+      end
       if errors.empty?
         @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
       else
@@ -140,10 +161,19 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
   end
 
   def check_connection!
-    @client.get_engine(@engine)
+    if connected_to_swiftype?
+      @client.get_engine(@engine)
+    else
+      res = @client.list_engines({:page_size => 1})
+      raise "Received HTTP error code #{res.status}" unless res.status == 200
+    end
   end
 
   def path_is_set?
     original_params.key?("path")
   end
+
+  def connected_to_swiftype?
+    @use_old_client
+  end
 end

data/logstash-integration-elastic_enterprise_search.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-integration-elastic_enterprise_search'
-  s.version         = '2.1.2'
+  s.version         = '2.2.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Elastic Enterprise Search - output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -27,6 +27,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
+  s.add_runtime_dependency "elastic-enterprise-search", '~>7.16.0'
   s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
+  s.add_runtime_dependency "logstash-mixin-deprecation_logger_support", '~>1.0'
   s.add_development_dependency "logstash-devutils"
 end

data/spec/integration/outputs/elastic_app_search_spec.rb

@@ -13,16 +13,16 @@ describe "indexing against running App Search", :integration => true do
 
   let(:config) do
     {
-      "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+      "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
       "engine" => engine_name,
-      "url" => "http://appsearch:3002"
+      "url" => "http://enterprise_search:3002"
     }
   end
 
   subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
 
   before(:each) do
-    create_engine(engine_name, "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+    create_engine(engine_name, "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
   end
 
   private
@@ -37,8 +37,8 @@ describe "indexing against running App Search", :integration => true do
   describe "search and private keys are configured" do
     let(:api_key_settings) do
       {
-        :private => ENV['APPSEARCH_PRIVATE_KEY'],
-        :search => ENV['APPSEARCH_SEARCH_KEY']
+        :private => ENV['APP_SEARCH_PRIVATE_KEY'],
+        :search => ENV['APP_SEARCH_SEARCH_KEY']
       }
     end
 
@@ -51,9 +51,9 @@ describe "indexing against running App Search", :integration => true do
   describe "register" do
     let(:config) do
       {
-        "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+        "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
         "engine" => "%{engine_name_field}",
-        "url" => "http://appsearch:3002"
+        "url" => "http://enterprise_search:3002"
       }
     end
 
@@ -89,9 +89,9 @@ describe "indexing against running App Search", :integration => true do
       context "using sprintf-ed engine" do
         let(:config) do
           {
-            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
             "engine" => "%{engine_name_field}",
-            "url" => "http://appsearch:3002"
+            "url" => "http://enterprise_search:3002"
           }
         end
 
@@ -134,15 +134,15 @@ describe "indexing against running App Search", :integration => true do
       context "multiple sprintf engines" do
         let(:config) do
           {
-            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
             "engine" => "%{engine_name_field}",
-            "url" => "http://appsearch:3002"
+            "url" => "http://enterprise_search:3002"
           }
         end
 
         it "all should be indexed" do
-         create_engine('testengin1', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
-         create_engine('testengin2', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+         create_engine('testengin1', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
+         create_engine('testengin2', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
          events = generate_events(100, 'testengin1')
          events += generate_events(100, 'testengin2')
          events.shuffle!

data/spec/integration/outputs/elastic_workplace_search_spec.rb

@@ -8,8 +8,8 @@ require "base64"
 
 describe "indexing against running Workplace Search", :integration => true do
 
-  let(:url) { ENV['APPSEARCH_URL'] }
-  let(:auth) { Base64.strict_encode64("#{ENV['APPSEARCH_USERNAME']}:#{ENV['AS_PASSWORD']}") }
+  let(:url) { ENV['ENTERPRISE_SEARCH_URL'] }
+  let(:auth) { Base64.strict_encode64("#{ENV['ENTERPRISE_SEARCH_USERNAME']}:#{ENV['ENTERPRISE_SEARCH_PASSWORD']}")}
   let(:source) do
     response = Faraday.post(
       "#{url}/ws/org/sources/form_create",

data/spec/unit/outputs/appsearch_spec.rb

@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elastic_app_search"
 require "logstash/codecs/plain"
 require "logstash/event"
+require "elastic-app-search"
 
 describe LogStash::Outputs::ElasticAppSearch do
   let(:sample_event) { LogStash::Event.new }
@@ -20,6 +21,19 @@ describe LogStash::Outputs::ElasticAppSearch do
       it "does not raise an error" do
         expect { subject.register }.to_not raise_error
       end
+      it "configures the Swiftype client" do
+        subject.register
+        client = subject.instance_variable_get(:@client)
+        expect(client.class).to eq(Elastic::AppSearch::Client)
+      end
+    end
+    context "when path is configured" do
+      let(:config) { { "api_key" => api_key, "engine" => engine, "path" => "/v1", "url" => "http://localhost:9300" } }
+      it "configures the Swiftype client" do
+        subject.register
+        client = subject.instance_variable_get(:@client)
+        expect(client.class).to eq(Elastic::AppSearch::Client)
+      end
     end
     context "when host and path is configured" do
       let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine, "path" => "/v1" } }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-elastic_enterprise_search
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-07 00:00:00.000000000 Z
+date: 2022-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 7.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.16.0
+  name: elastic-enterprise-search
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.16.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -66,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -124,8 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Integration with Elastic Enterprise Search - output plugins