logstash-integration-elastic_enterprise_search 2.1.0 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d1202fb961af64675edfe8d9deb5a5496a6883a4ba4f73de08b8c53229a78d8
-  data.tar.gz: a69156a70dd56f29ad1ba16330b31e81725f03679f5ec7ed2b4bf0e6d3220ad0
+  metadata.gz: 2f0c1374a08bc723fd7452e115e7372c8e0bb416c6e7e2e6d10769b15b571b7c
+  data.tar.gz: 53debfd033dae11b5d5c80809c51c8a6db5997c80f5eeb3901414932c0d48e17
 SHA512:
-  metadata.gz: 19e16ac5f649cb599b67394558a998393adf9d84331af212d7567f614b290832669aab601378ad7ca48472f3ab6bce1167e1edfa53bdcb378ae9435413fa8964
-  data.tar.gz: 5ce52eb632f0897c9af62e53a02db844142b8864908da08dd6937328a635ded2c4cb11e18d0f9a1861a2f8c9aa04acb0c29d93386647e81b612ac03ec9c151b8
+  metadata.gz: dcb96b16bafc1b85a0e4e375925b8d6ad5e6f3366044e1ea13b4318ab0c5b75b84d342fe76077ddcc30f4b0423f0e47028b474d2012e34570357a07ad9ff94e8
+  data.tar.gz: fab4c000386cca49902de6f11aa392d95032b4fd7897826fc7515dac40012315c2356d1c624485b4689faf229b7455176845ac40cdf3c298e596ad517a545a93

data/CHANGELOG.md

@@ -1,11 +1,26 @@
+## 2.2.1
+ - Fix, change implementation of connectivity check method to be compatible with version `v8.0+` of Workplace Search [#16](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/16) 
+
+## 2.2.0
+ - Feature, switch the connection library to elastic-enterprise-search [#3](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/3)
+ - [DOC] Added required parameters to Workplace Search example snippet and describe little better what's expected in url parameter [#11](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/11)
+
+## 2.1.2
+ - [DOC] Fix typos in App Search and Workplace Search [#9](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
+ - [DOC] Add links to Elastic App Search and Elastic Workplace Search solution pages [#10](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9) 
+
+## 2.1.1
+ - [DOC] Added the integration attribute and include statement for the integration plugin header to finalize hooking up integration docs [#8](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/8)
+ - [DOC] Added live link to output-elastic_workplace_search doc [#6](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/6)
+
 ## 2.1.0
-- Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
-- [DOC] Updates output-elastic_app_search documentation to add an integration attribute and include the integration plugin header [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
+ - Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
+ - [DOC] Added the integration attribute and include statement for the integration plugin header to output-elastic_app_search doc [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
 
 
 ## 2.0.0
-- Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
-  previous AppSearch Output plugin codebase; 
-  independent changelogs for previous versions can be found:
-   - [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
+ - Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
+   previous AppSearch Output plugin codebase; 
+   independent changelogs for previous versions can be found:
+    - [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
 

data/DEVELOPER.md

@@ -1,6 +1,7 @@
 # logstash-integration-elastic_enterprise_search
-Elastic Enterprise Search integration for Logstash, including AppSearch and WorkplaceSearch output plugins
+Elastic Enterprise Search integration for Logstash, including App Search and Workplace Search output plugins
 
 # Dependencies
- * elastic-app-search
+ * elastic-enterprise-search
+ * elastic-app-search  
  * elastic-workplace-search

data/lib/logstash/outputs/elastic_app_search.rb

@@ -1,8 +1,12 @@
 # encoding: utf-8
 require "logstash/outputs/base"
 require "elastic-app-search"
+require "elastic-enterprise-search"
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
+  include LogStash::PluginMixins::DeprecationLoggerSupport
+
   config_name "elastic_app_search"
 
   # The name of the search engine you created in App Search, an information
@@ -49,16 +53,25 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
 
   public
   def register
+    @use_old_client = false
     if @host.nil? && @url.nil?
       raise ::LogStash::ConfigurationError.new("Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
     elsif @host && @url
-      raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
+      raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed ot Elastic Enterprise Search) or \"host\" (for SaaS).")
     elsif @host && path_is_set?  # because path has a default value we need extra work to if the user set it
       raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
     elsif @host
+      @deprecation_logger.deprecated("Deprecated service usage, the `host` setting will be removed when Swiftype AppSearch service is shutdown")
+      @use_old_client = true
       @client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
     elsif @url
-      @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
+      if path_is_set?
+        @deprecation_logger.deprecated("Deprecated service usage, the `path` setting will be removed when Swiftype AppSearch service is shutdown")
+        @use_old_client = true
+        @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
+      else
+        @client = Elastic::EnterpriseSearch::AppSearch::Client.new(:host => @url, :http_auth => @api_key.value, :external_url => @url)
+      end
     end
     check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
   rescue => e
@@ -77,7 +90,7 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
     events.each_slice(100) do |events|
       batch = format_batch(events)
       if @logger.trace?
-        @logger.trace("Sending bulk to AppSearch", :size => batch.size, :data => batch.inspect)
+        @logger.trace("Sending bulk to App Search", :size => batch.size, :data => batch.inspect)
       end
       index(batch)
     end
@@ -117,11 +130,15 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
         if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
           raise "Cannot resolve engine field name #{@engine} from event"
         end
-        response = @client.index_documents(resolved_engine, documents)
+        if connected_to_swiftype?
+          response = @client.index_documents(resolved_engine, documents)
+        else
+          response = @client.index_documents(resolved_engine, {:documents => documents})
+        end
         report(documents, response)
       rescue => e
         @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
-                      :resolved_engine => resolved_engine)
+                      :resolved_engine => resolved_engine, :backtrace => e.backtrace)
         sleep(1)
         retry
       end
@@ -130,7 +147,11 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
 
   def report(documents, response)
     documents.each_with_index do |document, i|
-      errors = response[i]["errors"]
+      if connected_to_swiftype?
+        errors = response[i]["errors"]
+      else
+        errors = response.body[i]["errors"]
+      end
       if errors.empty?
         @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
       else
@@ -140,10 +161,19 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
   end
 
   def check_connection!
-    @client.get_engine(@engine)
+    if connected_to_swiftype?
+      @client.get_engine(@engine)
+    else
+      res = @client.list_engines({:page_size => 1})
+      raise "Received HTTP error code #{res.status}" unless res.status == 200
+    end
   end
 
   def path_is_set?
     original_params.key?("path")
   end
+
+  def connected_to_swiftype?
+    @use_old_client
+  end
 end

data/lib/logstash/outputs/elastic_workplace_search.rb

@@ -101,6 +101,7 @@ class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
   end
 
   def check_connection!
-    @client.list_all_permissions(@source)
+    # This is the preferred way to check compatibility across different versions of Workplace Search service.
+    @client.index_documents(@source, {:documents => []})
   end
 end

data/logstash-integration-elastic_enterprise_search.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-integration-elastic_enterprise_search'
-  s.version         = '2.1.0'
+  s.version         = '2.2.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Elastic Enterprise Search - output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -27,6 +27,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
+  s.add_runtime_dependency "elastic-enterprise-search", '~>7.16.0'
   s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
+  s.add_runtime_dependency "logstash-mixin-deprecation_logger_support", '~>1.0'
   s.add_development_dependency "logstash-devutils"
 end

data/spec/integration/outputs/elastic_app_search_spec.rb

@@ -5,7 +5,7 @@ require "logstash/codecs/plain"
 require "logstash/event"
 require "json"
 
-describe "indexing against running AppSearch", :integration => true do
+describe "indexing against running App Search", :integration => true do
 
   let(:engine_name) do
     (0...10).map { ('a'..'z').to_a[rand(26)] }.join
@@ -13,16 +13,16 @@ describe "indexing against running AppSearch", :integration => true do
 
   let(:config) do
     {
-      "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+      "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
       "engine" => engine_name,
-      "url" => "http://appsearch:3002"
+      "url" => "http://enterprise_search:3002"
     }
   end
 
   subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
 
   before(:each) do
-    create_engine(engine_name, "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+    create_engine(engine_name, "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
   end
 
   private
@@ -37,8 +37,8 @@ describe "indexing against running AppSearch", :integration => true do
   describe "search and private keys are configured" do
     let(:api_key_settings) do
       {
-        :private => ENV['APPSEARCH_PRIVATE_KEY'],
-        :search => ENV['APPSEARCH_SEARCH_KEY']
+        :private => ENV['APP_SEARCH_PRIVATE_KEY'],
+        :search => ENV['APP_SEARCH_SEARCH_KEY']
       }
     end
 
@@ -51,9 +51,9 @@ describe "indexing against running AppSearch", :integration => true do
   describe "register" do
     let(:config) do
       {
-        "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+        "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
         "engine" => "%{engine_name_field}",
-        "url" => "http://appsearch:3002"
+        "url" => "http://enterprise_search:3002"
       }
     end
 
@@ -89,9 +89,9 @@ describe "indexing against running AppSearch", :integration => true do
       context "using sprintf-ed engine" do
         let(:config) do
           {
-            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
             "engine" => "%{engine_name_field}",
-            "url" => "http://appsearch:3002"
+            "url" => "http://enterprise_search:3002"
           }
         end
 
@@ -134,15 +134,15 @@ describe "indexing against running AppSearch", :integration => true do
       context "multiple sprintf engines" do
         let(:config) do
           {
-            "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+            "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
             "engine" => "%{engine_name_field}",
-            "url" => "http://appsearch:3002"
+            "url" => "http://enterprise_search:3002"
           }
         end
 
         it "all should be indexed" do
-         create_engine('testengin1', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
-         create_engine('testengin2', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+         create_engine('testengin1', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
+         create_engine('testengin2', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
          events = generate_events(100, 'testengin1')
          events += generate_events(100, 'testengin2')
          events.shuffle!

data/spec/integration/outputs/elastic_workplace_search_spec.rb

@@ -8,25 +8,68 @@ require "base64"
 
 describe "indexing against running Workplace Search", :integration => true do
 
-  let(:url) { ENV['APPSEARCH_URL'] }
-  let(:auth) { Base64.strict_encode64("#{ENV['APPSEARCH_USERNAME']}:#{ENV['AS_PASSWORD']}") }
+  def is_version7?
+    ENV['ELASTIC_STACK_VERSION'].strip.start_with?("7")
+  end
+
+  let(:url) { ENV['ENTERPRISE_SEARCH_URL'] }
+  let(:auth) { Base64.strict_encode64("#{ENV['ENTERPRISE_SEARCH_USERNAME']}:#{ENV['ENTERPRISE_SEARCH_PASSWORD']}")}
   let(:source) do
+    if is_version7?
+      response = Faraday.get(
+        "#{url}/api/ws/v1/whoami",
+        {"get_token" => true},
+        {"Content-Type" => "application/json",
+        "Accept" => "application/json",
+        "Authorization" => "Basic #{auth}"}
+      )
+      JSON.load(response.body)
+    else
+      # Workplace Search v8.0+ provides the api_tokens API to create or retrieve
+      # the key to be use as access_token
+      conn = Faraday.new(url: url)
+      conn.basic_auth(ENV['ENTERPRISE_SEARCH_USERNAME'], ENV['ENTERPRISE_SEARCH_PASSWORD'])
+      response = conn.post('/ws/org/api_tokens',
+                            '{"name":"ls-integration-test-key"}',
+                            {"Content-Type" => "application/json", "Accept" => "application/json"})
+      create_response_json = JSON.load(response.body)
+      if create_response_json.has_key?("errors") && create_response_json["errors"].include?("Name is already taken")
+        # when a key with the name already exists, retrieve it
+        response = conn.get('/ws/org/api_tokens', nil,  {"Content-Type" => "application/json", "Accept" => "application/json"})
+        retrieve_token_response_json = JSON.load(response.body)
+        response_json = retrieve_token_response_json["results"].find {|res| res["id"] == "ls-integration-test-key"}
+      else
+        response_json = create_response_json
+      end
+
+      conn.close
+      response_json
+    end
+  end
+  let(:access_token) do
+    if is_version7?
+      source.fetch("access_token")
+    else
+      source.fetch("key")
+    end
+  end
+  let(:source_id) do
     response = Faraday.post(
-      "#{url}/ws/org/sources/form_create",
-      JSON.dump("service_type" => "custom", "name" => "whatever"),
-      "Content-Type" => "application/json",
-      "Accept" => "application/json",
-      "Authorization" => "Basic #{auth}"
-    )
-    JSON.load(response.body)
+          "#{url}/api/ws/v1/sources",
+           JSON.dump("service_type" => "custom", "name" => "whatever"),
+          {"Content-Type" => "application/json",
+          "Accept" => "application/json",
+          "Authorization" => "Bearer #{access_token}"}
+        )
+    source_response_json = JSON.load(response.body)
+    source_response_json.fetch("id")
   end
-  let(:source_id) { source.fetch("id") }
 
   let(:config) do
     {
       "url" => url,
       "source" => source_id,
-      "access_token" => source.fetch("accessToken")
+      "access_token" => access_token
     }
   end
 

data/spec/unit/outputs/appsearch_spec.rb

@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elastic_app_search"
 require "logstash/codecs/plain"
 require "logstash/event"
+require "elastic-app-search"
 
 describe LogStash::Outputs::ElasticAppSearch do
   let(:sample_event) { LogStash::Event.new }
@@ -20,6 +21,19 @@ describe LogStash::Outputs::ElasticAppSearch do
       it "does not raise an error" do
         expect { subject.register }.to_not raise_error
       end
+      it "configures the Swiftype client" do
+        subject.register
+        client = subject.instance_variable_get(:@client)
+        expect(client.class).to eq(Elastic::AppSearch::Client)
+      end
+    end
+    context "when path is configured" do
+      let(:config) { { "api_key" => api_key, "engine" => engine, "path" => "/v1", "url" => "http://localhost:9300" } }
+      it "configures the Swiftype client" do
+        subject.register
+        client = subject.instance_variable_get(:@client)
+        expect(client.class).to eq(Elastic::AppSearch::Client)
+      end
     end
     context "when host and path is configured" do
       let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine, "path" => "/v1" } }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-elastic_enterprise_search
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-18 00:00:00.000000000 Z
+date: 2022-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 7.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.16.0
+  name: elastic-enterprise-search
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.16.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -66,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -124,8 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Integration with Elastic Enterprise Search - output plugins