logstash-integration-elastic_enterprise_search 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe83963cbb7d00dffdf8b52106d2db2e34f8b5389c891a7a1db7f939657c6ed2
-  data.tar.gz: 21be61a90719d99c958a9d4eed07a67a4d5ea22d6fbcf5cb8531a3af5e036c01
+  metadata.gz: 0d1202fb961af64675edfe8d9deb5a5496a6883a4ba4f73de08b8c53229a78d8
+  data.tar.gz: a69156a70dd56f29ad1ba16330b31e81725f03679f5ec7ed2b4bf0e6d3220ad0
 SHA512:
-  metadata.gz: f4fece8fe3bec522750956526422d44da6b0ea1b2bcf42d01ca992f5835f67ab5e173e9543442da453f97dc68c78cf87fab8a7370925d7696e68069eeb20f16d
-  data.tar.gz: 88ba3212c15e0f84a2023472a80b4acbb81663db628d6ef209cba3295ec4310a78b06e21c1c3a31a552ea08e249faf04a6fc8efa5f3f52998dddd1557a8b2eb6
+  metadata.gz: 19e16ac5f649cb599b67394558a998393adf9d84331af212d7567f614b290832669aab601378ad7ca48472f3ab6bce1167e1edfa53bdcb378ae9435413fa8964
+  data.tar.gz: 5ce52eb632f0897c9af62e53a02db844142b8864908da08dd6937328a635ded2c4cb11e18d0f9a1861a2f8c9aa04acb0c29d93386647e81b612ac03ec9c151b8

data/CHANGELOG.md

@@ -1,5 +1,9 @@
-## 2.0.0
+## 2.1.0
+- Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
+- [DOC] Updates output-elastic_app_search documentation to add an integration attribute and include the integration plugin header [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
+
 
+## 2.0.0
 - Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
   previous AppSearch Output plugin codebase; 
   independent changelogs for previous versions can be found:

data/DEVELOPER.md

@@ -2,4 +2,5 @@
 Elastic Enterprise Search integration for Logstash, including AppSearch and WorkplaceSearch output plugins
 
 # Dependencies
- * elastic-app-search
+ * elastic-app-search
+ * elastic-workplace-search

data/README.md

@@ -97,4 +97,3 @@ Programming is not a required skill. Whatever you've seen about open source and
 It is more important to the community that you are able to contribute.
 
 For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
->>>>>>> 418bfff... Rought output plugin creation

data/lib/logstash/outputs/elastic_app_search.rb

@@ -5,12 +5,44 @@ require "elastic-app-search"
 class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
   config_name "elastic_app_search"
 
+  # The name of the search engine you created in App Search, an information
+  # repository that includes the indexed document records.
+  # The `engine` field supports
+  # {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
+  # allow the engine name to be derived from a field value from each event, for
+  # example `engine-%{engine_name}`.
+  #
+  # Invalid engine names cause ingestion to stop until the field value can be resolved into a valid engine name.
+  # This situation can happen if the interpolated field value resolves to a value without a matching engine,
+  # or, if the field is missing from the event and cannot be resolved at all.
   config :engine, :validate => :string, :required => true
+
+  # The hostname of the App Search API that is associated with your App Search account.
+  # Set this when using the https://www.elastic.co/cloud/app-search-service
   config :host, :validate => :string
+
+  # The value of the API endpoint in the form of a URL. Note: The value of the of the `path` setting will be will be appended to this URL.
+  # Set this when using the https://www.elastic.co/downloads/app-search
   config :url, :validate => :string
+
+  # The private API Key with write permissions. Visit the https://app.swiftype.com/as/credentials
+  # in the App Search dashboard to find the key associated with your account.
   config :api_key, :validate => :password, :required => true
+
+  # Where to move the value from the `@timestamp` field.
+  #
+  # All Logstash events contain a `@timestamp` field.
+  # App Search doesn't support fields starting with `@timestamp`, and
+  # by default, the `@timestamp` field will be deleted.
+  #
+  # To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
   config :timestamp_destination, :validate => :string
+
+  # The id for app search documents. This can be an interpolated value
+  # like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
   config :document_id, :validate => :string
+
+  # The path that is appended to the `url` parameter when connecting to a https://www.elastic.co/downloads/app-search
   config :path, :validate => :string, :default => "/api/as/v1/"
 
   ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/

data/lib/logstash/outputs/elastic_workplace_search.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "elastic-workplace-search"
+
+class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
+  config_name "elastic_workplace_search"
+
+  # The value of the API endpoint in the form of a URL.
+  config :url, :validate => :string, :required => true
+
+  # The ID of the source you created in Workplace Search.
+  # The `source` field supports
+  # {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
+  # allow the source ID to be derived from a field value from each event, for
+  # example `%{source_id}`.
+  #
+  # Invalid source IDs cause ingestion to stop until the field value can be resolved into a valid source ID.
+  # This situation can happen if the interpolated field value resolves to a value without a matching source,
+  # or, if the field is missing from the event and cannot be resolved at all.
+  config :source, :validate => :string, :required => true
+
+  # The source access token. Visit the source overview page in the Workplace Search dashboard
+  # to find the token associated with your source.
+  config :access_token, :validate => :password, :required => true
+
+  # Where to move the value from the `@timestamp` field.
+  #
+  # All Logstash events contain a `@timestamp` field.
+  # Workplace Search doesn't support fields starting with `@timestamp`, and
+  # by default, the `@timestamp` field will be deleted.
+  #
+  # To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
+  config :timestamp_destination, :validate => :string
+
+  # The id for workplace search documents. This can be an interpolated value
+  # like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
+  config :document_id, :validate => :string
+
+  public
+  def register
+    Elastic::WorkplaceSearch.endpoint = "#{@url.chomp('/')}/api/ws/v1/"
+    @client = Elastic::WorkplaceSearch::Client.new(:access_token => @access_token.value)
+    check_connection!
+  rescue Elastic::WorkplaceSearch::InvalidCredentials
+    raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 401. Please check your credentials")
+  rescue Elastic::WorkplaceSearch::NonExistentRecord
+    raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 404. Please check if url '#{@url}' is correct and you've created a source with ID '#{@source}'")
+  rescue => e
+    raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. #{e.message}")
+  end
+
+  public
+  def multi_receive(events)
+    # because Workplace Search has a limit of 100 documents per bulk
+    events.each_slice(100) do |events|
+      batch = format_batch(events)
+      if @logger.trace?
+        @logger.trace("Sending bulk to Workplace Search", :size => batch.size, :data => batch.inspect)
+      end
+      index(batch)
+    end
+  end
+
+  private
+  def format_batch(events)
+    events.map do |event|
+      doc = event.to_hash
+      # we need to remove default fields that start with "@"
+      # since Elastic Workplace Search doesn't accept them
+      if @timestamp_destination
+        doc[@timestamp_destination] = doc.delete("@timestamp")
+      else # delete it
+        doc.delete("@timestamp")
+      end
+      if @document_id
+        doc["id"] = event.sprintf(@document_id)
+      end
+      doc.delete("@version")
+      doc
+    end
+  end
+
+  def index(documents)
+    response = @client.index_documents(@source, documents)
+    report(documents, response)
+  rescue => e
+    @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message)
+    sleep(1)
+    retry
+  end
+
+  def report(documents, response)
+    documents.each_with_index do |document, i|
+      errors = response["results"][i]["errors"]
+      if errors.empty?
+        @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
+      else
+        @logger.warn("Document failed to index. Dropping..", :document => document, :errors => errors.to_a)
+      end
+    end
+  end
+
+  def check_connection!
+    @client.list_all_permissions(@source)
+  end
+end

data/logstash-integration-elastic_enterprise_search.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-integration-elastic_enterprise_search'
-  s.version         = '2.0.0'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Elastic Enterprise Search - output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -20,12 +20,13 @@ Gem::Specification.new do |s|
   s.metadata = {
       "logstash_plugin"     => "true",
       "logstash_group"      => "integration",
-      "integration_plugins" => "logstash-output-elastic_app_search"
+      "integration_plugins" => "logstash-output-elastic_app_search, logstash-output-elastic_workplace_search"
   }
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
+  s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
   s.add_development_dependency "logstash-devutils"
 end

data/spec/integration/outputs/elastic_workplace_search_spec.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elastic_workplace_search"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+require "base64"
+
+describe "indexing against running Workplace Search", :integration => true do
+
+  let(:url) { ENV['APPSEARCH_URL'] }
+  let(:auth) { Base64.strict_encode64("#{ENV['APPSEARCH_USERNAME']}:#{ENV['AS_PASSWORD']}") }
+  let(:source) do
+    response = Faraday.post(
+      "#{url}/ws/org/sources/form_create",
+      JSON.dump("service_type" => "custom", "name" => "whatever"),
+      "Content-Type" => "application/json",
+      "Accept" => "application/json",
+      "Authorization" => "Basic #{auth}"
+    )
+    JSON.load(response.body)
+  end
+  let(:source_id) { source.fetch("id") }
+
+  let(:config) do
+    {
+      "url" => url,
+      "source" => source_id,
+      "access_token" => source.fetch("accessToken")
+    }
+  end
+
+  subject(:workplace_search_output) { LogStash::Outputs::ElasticWorkplaceSearch.new(config) }
+
+  before(:each) { workplace_search_output.register }
+
+  describe "single event" do
+    let(:event) { LogStash::Event.new("message" => "an event to index") }
+
+    it "should be indexed" do
+      workplace_search_output.multi_receive([event])
+
+      results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+        attempt_response = execute_search_call
+        expect(attempt_response.status).to eq(200)
+        parsed_resp = JSON.parse(attempt_response.body)
+        expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
+        parsed_resp["results"]
+      end
+      expect(results.first.fetch("message")).to eq "an event to index"
+    end
+  end
+
+  describe "multiple events" do
+    let(:events) { generate_events(200) } #2 times the slice size used to batch
+
+    it "all should be indexed" do
+      workplace_search_output.multi_receive(events)
+      results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+        attempt_response = execute_search_call
+        expect(attempt_response.status).to eq(200)
+        parsed_resp = JSON.parse(attempt_response.body)
+        expect(parsed_resp.dig("meta", "page", "total_results")).to eq(200)
+        parsed_resp["results"]
+      end
+      expect(results.first.fetch("message")).to start_with("an event to index")
+    end
+  end
+
+  private
+  def execute_search_call
+    Faraday.post(
+      "#{url}/ws/org/sources/#{source_id}/documents",
+      nil,
+      "Accept" => "application/json",
+      "Authorization" => "Basic #{auth}"
+    )
+  end
+
+  def generate_events(num_events)
+    (1..num_events).map { |i| LogStash::Event.new("message" => "an event to index #{i}")}
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-elastic_enterprise_search
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-05 00:00:00.000000000 Z
+date: 2021-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 7.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  name: elastic-workplace-search
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -82,8 +96,10 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/elastic_app_search.rb
+- lib/logstash/outputs/elastic_workplace_search.rb
 - logstash-integration-elastic_enterprise_search.gemspec
-- spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/elastic_app_search_spec.rb
+- spec/integration/outputs/elastic_workplace_search_spec.rb
 - spec/unit/outputs/appsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -91,7 +107,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: integration
-  integration_plugins: logstash-output-elastic_app_search
+  integration_plugins: logstash-output-elastic_app_search, logstash-output-elastic_workplace_search
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -114,5 +130,6 @@ signing_key:
 specification_version: 4
 summary: Integration with Elastic Enterprise Search - output plugins
 test_files:
-- spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/elastic_app_search_spec.rb
+- spec/integration/outputs/elastic_workplace_search_spec.rb
 - spec/unit/outputs/appsearch_spec.rb