logstash-integration-elastic_enterprise_search 2.0.0 → 2.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fe83963cbb7d00dffdf8b52106d2db2e34f8b5389c891a7a1db7f939657c6ed2
4
- data.tar.gz: 21be61a90719d99c958a9d4eed07a67a4d5ea22d6fbcf5cb8531a3af5e036c01
3
+ metadata.gz: 6276fe38b651d58dcfa68a99d6d6de58f4b7a35d4e5cf640f996cbd7b681a7ad
4
+ data.tar.gz: 373697b1c10c3c51af094d0415e407033ea1d987d776f4c0e0a83146b998f2cf
5
5
  SHA512:
6
- metadata.gz: f4fece8fe3bec522750956526422d44da6b0ea1b2bcf42d01ca992f5835f67ab5e173e9543442da453f97dc68c78cf87fab8a7370925d7696e68069eeb20f16d
7
- data.tar.gz: 88ba3212c15e0f84a2023472a80b4acbb81663db628d6ef209cba3295ec4310a78b06e21c1c3a31a552ea08e249faf04a6fc8efa5f3f52998dddd1557a8b2eb6
6
+ metadata.gz: 898a93fb3d0516b4b6f2a0eb358a29809e581a56ed7c5ca3be21e95f778dd4e63b81e615dbe9bfb96a22cc1439d3f840f0e574a4b56de4e08de3a5eff36ccce9
7
+ data.tar.gz: 9f4edd07607d41ee54ec36141dd0b009ae8eb5c24364b05a200b419e2efdd800ed57694f10c150e56c1186ee54639fff31361a6c72e9ecff44fb431f251c82b9
data/CHANGELOG.md CHANGED
@@ -1,7 +1,24 @@
1
- ## 2.0.0
2
1
 
3
- - Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
4
- previous AppSearch Output plugin codebase;
5
- independent changelogs for previous versions can be found:
6
- - [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
2
+ ## 2.2.0
3
+ - Feature, switch the connection library to elastic-enterprise-search [#3](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/3)
4
+ - [DOC] Added required parameters to Workplace Search example snippet and describe little better what's expected in url parameter [#11](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/11)
5
+
6
+ ## 2.1.2
7
+ - [DOC] Fix typos in App Search and Workplace Search [#9](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
8
+ - [DOC] Add links to Elastic App Search and Elastic Workplace Search solution pages [#10](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
9
+
10
+ ## 2.1.1
11
+ - [DOC] Added the integration attribute and include statement for the integration plugin header to finalize hooking up integration docs [#8](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/8)
12
+ - [DOC] Added live link to output-elastic_workplace_search doc [#6](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/6)
13
+
14
+ ## 2.1.0
15
+ - Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
16
+ - [DOC] Added the integration attribute and include statement for the integration plugin header to output-elastic_app_search doc [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
17
+
18
+
19
+ ## 2.0.0
20
+ - Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
21
+ previous AppSearch Output plugin codebase;
22
+ independent changelogs for previous versions can be found:
23
+ - [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
7
24
 
data/DEVELOPER.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # logstash-integration-elastic_enterprise_search
2
- Elastic Enterprise Search integration for Logstash, including AppSearch and WorkplaceSearch output plugins
2
+ Elastic Enterprise Search integration for Logstash, including App Search and Workplace Search output plugins
3
3
 
4
4
  # Dependencies
5
- * elastic-app-search
5
+ * elastic-enterprise-search
6
+ * elastic-app-search
7
+ * elastic-workplace-search
data/README.md CHANGED
@@ -97,4 +97,3 @@ Programming is not a required skill. Whatever you've seen about open source and
97
97
  It is more important to the community that you are able to contribute.
98
98
 
99
99
  For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
100
- >>>>>>> 418bfff... Rought output plugin creation
@@ -1,32 +1,77 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/outputs/base"
3
3
  require "elastic-app-search"
4
+ require "elastic-enterprise-search"
5
+ require 'logstash/plugin_mixins/deprecation_logger_support'
4
6
 
5
7
  class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
8
+ include LogStash::PluginMixins::DeprecationLoggerSupport
9
+
6
10
  config_name "elastic_app_search"
7
11
 
12
+ # The name of the search engine you created in App Search, an information
13
+ # repository that includes the indexed document records.
14
+ # The `engine` field supports
15
+ # {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
16
+ # allow the engine name to be derived from a field value from each event, for
17
+ # example `engine-%{engine_name}`.
18
+ #
19
+ # Invalid engine names cause ingestion to stop until the field value can be resolved into a valid engine name.
20
+ # This situation can happen if the interpolated field value resolves to a value without a matching engine,
21
+ # or, if the field is missing from the event and cannot be resolved at all.
8
22
  config :engine, :validate => :string, :required => true
23
+
24
+ # The hostname of the App Search API that is associated with your App Search account.
25
+ # Set this when using the https://www.elastic.co/cloud/app-search-service
9
26
  config :host, :validate => :string
27
+
28
+ # The value of the API endpoint in the form of a URL. Note: The value of the of the `path` setting will be will be appended to this URL.
29
+ # Set this when using the https://www.elastic.co/downloads/app-search
10
30
  config :url, :validate => :string
31
+
32
+ # The private API Key with write permissions. Visit the https://app.swiftype.com/as/credentials
33
+ # in the App Search dashboard to find the key associated with your account.
11
34
  config :api_key, :validate => :password, :required => true
35
+
36
+ # Where to move the value from the `@timestamp` field.
37
+ #
38
+ # All Logstash events contain a `@timestamp` field.
39
+ # App Search doesn't support fields starting with `@timestamp`, and
40
+ # by default, the `@timestamp` field will be deleted.
41
+ #
42
+ # To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
12
43
  config :timestamp_destination, :validate => :string
44
+
45
+ # The id for app search documents. This can be an interpolated value
46
+ # like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
13
47
  config :document_id, :validate => :string
48
+
49
+ # The path that is appended to the `url` parameter when connecting to a https://www.elastic.co/downloads/app-search
14
50
  config :path, :validate => :string, :default => "/api/as/v1/"
15
51
 
16
52
  ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
17
53
 
18
54
  public
19
55
  def register
56
+ @use_old_client = false
20
57
  if @host.nil? && @url.nil?
21
58
  raise ::LogStash::ConfigurationError.new("Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
22
59
  elsif @host && @url
23
- raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
60
+ raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed ot Elastic Enterprise Search) or \"host\" (for SaaS).")
24
61
  elsif @host && path_is_set? # because path has a default value we need extra work to if the user set it
25
62
  raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
26
63
  elsif @host
64
+ @deprecation_logger.deprecated("Deprecated service usage, the `host` setting will be removed when Swiftype AppSearch service is shutdown")
65
+ @use_old_client = true
27
66
  @client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
28
67
  elsif @url
29
- @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
68
+ if path_is_set?
69
+ @deprecation_logger.deprecated("Deprecated service usage, the `path` setting will be removed when Swiftype AppSearch service is shutdown")
70
+ @use_old_client = true
71
+ @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
72
+ else
73
+ @client = Elastic::EnterpriseSearch::AppSearch::Client.new(:host => @url, :http_auth => @api_key.value, :external_url => @url)
74
+ end
30
75
  end
31
76
  check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
32
77
  rescue => e
@@ -45,7 +90,7 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
45
90
  events.each_slice(100) do |events|
46
91
  batch = format_batch(events)
47
92
  if @logger.trace?
48
- @logger.trace("Sending bulk to AppSearch", :size => batch.size, :data => batch.inspect)
93
+ @logger.trace("Sending bulk to App Search", :size => batch.size, :data => batch.inspect)
49
94
  end
50
95
  index(batch)
51
96
  end
@@ -85,11 +130,15 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
85
130
  if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
86
131
  raise "Cannot resolve engine field name #{@engine} from event"
87
132
  end
88
- response = @client.index_documents(resolved_engine, documents)
133
+ if connected_to_swiftype?
134
+ response = @client.index_documents(resolved_engine, documents)
135
+ else
136
+ response = @client.index_documents(resolved_engine, {:documents => documents})
137
+ end
89
138
  report(documents, response)
90
139
  rescue => e
91
140
  @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
92
- :resolved_engine => resolved_engine)
141
+ :resolved_engine => resolved_engine, :backtrace => e.backtrace)
93
142
  sleep(1)
94
143
  retry
95
144
  end
@@ -98,7 +147,11 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
98
147
 
99
148
  def report(documents, response)
100
149
  documents.each_with_index do |document, i|
101
- errors = response[i]["errors"]
150
+ if connected_to_swiftype?
151
+ errors = response[i]["errors"]
152
+ else
153
+ errors = response.body[i]["errors"]
154
+ end
102
155
  if errors.empty?
103
156
  @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
104
157
  else
@@ -108,10 +161,19 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
108
161
  end
109
162
 
110
163
  def check_connection!
111
- @client.get_engine(@engine)
164
+ if connected_to_swiftype?
165
+ @client.get_engine(@engine)
166
+ else
167
+ res = @client.list_engines({:page_size => 1})
168
+ raise "Received HTTP error code #{res.status}" unless res.status == 200
169
+ end
112
170
  end
113
171
 
114
172
  def path_is_set?
115
173
  original_params.key?("path")
116
174
  end
175
+
176
+ def connected_to_swiftype?
177
+ @use_old_client
178
+ end
117
179
  end
@@ -0,0 +1,106 @@
1
+ # encoding: utf-8
2
+ require "logstash/outputs/base"
3
+ require "elastic-workplace-search"
4
+
5
+ class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
6
+ config_name "elastic_workplace_search"
7
+
8
+ # The value of the API endpoint in the form of a URL.
9
+ config :url, :validate => :string, :required => true
10
+
11
+ # The ID of the source you created in Workplace Search.
12
+ # The `source` field supports
13
+ # {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
14
+ # allow the source ID to be derived from a field value from each event, for
15
+ # example `%{source_id}`.
16
+ #
17
+ # Invalid source IDs cause ingestion to stop until the field value can be resolved into a valid source ID.
18
+ # This situation can happen if the interpolated field value resolves to a value without a matching source,
19
+ # or, if the field is missing from the event and cannot be resolved at all.
20
+ config :source, :validate => :string, :required => true
21
+
22
+ # The source access token. Visit the source overview page in the Workplace Search dashboard
23
+ # to find the token associated with your source.
24
+ config :access_token, :validate => :password, :required => true
25
+
26
+ # Where to move the value from the `@timestamp` field.
27
+ #
28
+ # All Logstash events contain a `@timestamp` field.
29
+ # Workplace Search doesn't support fields starting with `@timestamp`, and
30
+ # by default, the `@timestamp` field will be deleted.
31
+ #
32
+ # To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
33
+ config :timestamp_destination, :validate => :string
34
+
35
+ # The id for workplace search documents. This can be an interpolated value
36
+ # like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
37
+ config :document_id, :validate => :string
38
+
39
+ public
40
+ def register
41
+ Elastic::WorkplaceSearch.endpoint = "#{@url.chomp('/')}/api/ws/v1/"
42
+ @client = Elastic::WorkplaceSearch::Client.new(:access_token => @access_token.value)
43
+ check_connection!
44
+ rescue Elastic::WorkplaceSearch::InvalidCredentials
45
+ raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 401. Please check your credentials")
46
+ rescue Elastic::WorkplaceSearch::NonExistentRecord
47
+ raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 404. Please check if url '#{@url}' is correct and you've created a source with ID '#{@source}'")
48
+ rescue => e
49
+ raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. #{e.message}")
50
+ end
51
+
52
+ public
53
+ def multi_receive(events)
54
+ # because Workplace Search has a limit of 100 documents per bulk
55
+ events.each_slice(100) do |events|
56
+ batch = format_batch(events)
57
+ if @logger.trace?
58
+ @logger.trace("Sending bulk to Workplace Search", :size => batch.size, :data => batch.inspect)
59
+ end
60
+ index(batch)
61
+ end
62
+ end
63
+
64
+ private
65
+ def format_batch(events)
66
+ events.map do |event|
67
+ doc = event.to_hash
68
+ # we need to remove default fields that start with "@"
69
+ # since Elastic Workplace Search doesn't accept them
70
+ if @timestamp_destination
71
+ doc[@timestamp_destination] = doc.delete("@timestamp")
72
+ else # delete it
73
+ doc.delete("@timestamp")
74
+ end
75
+ if @document_id
76
+ doc["id"] = event.sprintf(@document_id)
77
+ end
78
+ doc.delete("@version")
79
+ doc
80
+ end
81
+ end
82
+
83
+ def index(documents)
84
+ response = @client.index_documents(@source, documents)
85
+ report(documents, response)
86
+ rescue => e
87
+ @logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message)
88
+ sleep(1)
89
+ retry
90
+ end
91
+
92
+ def report(documents, response)
93
+ documents.each_with_index do |document, i|
94
+ errors = response["results"][i]["errors"]
95
+ if errors.empty?
96
+ @logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
97
+ else
98
+ @logger.warn("Document failed to index. Dropping..", :document => document, :errors => errors.to_a)
99
+ end
100
+ end
101
+ end
102
+
103
+ def check_connection!
104
+ @client.list_all_permissions(@source)
105
+ end
106
+ end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-integration-elastic_enterprise_search'
3
- s.version = '2.0.0'
3
+ s.version = '2.2.0'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Integration with Elastic Enterprise Search - output plugins"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
@@ -20,12 +20,15 @@ Gem::Specification.new do |s|
20
20
  s.metadata = {
21
21
  "logstash_plugin" => "true",
22
22
  "logstash_group" => "integration",
23
- "integration_plugins" => "logstash-output-elastic_app_search"
23
+ "integration_plugins" => "logstash-output-elastic_app_search, logstash-output-elastic_workplace_search"
24
24
  }
25
25
 
26
26
  # Gem dependencies
27
27
  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
28
28
  s.add_runtime_dependency "logstash-codec-plain"
29
29
  s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
30
+ s.add_runtime_dependency "elastic-enterprise-search", '~>7.16.0'
31
+ s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
32
+ s.add_runtime_dependency "logstash-mixin-deprecation_logger_support", '~>1.0'
30
33
  s.add_development_dependency "logstash-devutils"
31
34
  end
@@ -5,7 +5,7 @@ require "logstash/codecs/plain"
5
5
  require "logstash/event"
6
6
  require "json"
7
7
 
8
- describe "indexing against running AppSearch", :integration => true do
8
+ describe "indexing against running App Search", :integration => true do
9
9
 
10
10
  let(:engine_name) do
11
11
  (0...10).map { ('a'..'z').to_a[rand(26)] }.join
@@ -13,16 +13,16 @@ describe "indexing against running AppSearch", :integration => true do
13
13
 
14
14
  let(:config) do
15
15
  {
16
- "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
16
+ "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
17
17
  "engine" => engine_name,
18
- "url" => "http://appsearch:3002"
18
+ "url" => "http://enterprise_search:3002"
19
19
  }
20
20
  end
21
21
 
22
22
  subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
23
23
 
24
24
  before(:each) do
25
- create_engine(engine_name, "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
25
+ create_engine(engine_name, "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
26
26
  end
27
27
 
28
28
  private
@@ -37,8 +37,8 @@ describe "indexing against running AppSearch", :integration => true do
37
37
  describe "search and private keys are configured" do
38
38
  let(:api_key_settings) do
39
39
  {
40
- :private => ENV['APPSEARCH_PRIVATE_KEY'],
41
- :search => ENV['APPSEARCH_SEARCH_KEY']
40
+ :private => ENV['APP_SEARCH_PRIVATE_KEY'],
41
+ :search => ENV['APP_SEARCH_SEARCH_KEY']
42
42
  }
43
43
  end
44
44
 
@@ -51,9 +51,9 @@ describe "indexing against running AppSearch", :integration => true do
51
51
  describe "register" do
52
52
  let(:config) do
53
53
  {
54
- "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
54
+ "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
55
55
  "engine" => "%{engine_name_field}",
56
- "url" => "http://appsearch:3002"
56
+ "url" => "http://enterprise_search:3002"
57
57
  }
58
58
  end
59
59
 
@@ -89,9 +89,9 @@ describe "indexing against running AppSearch", :integration => true do
89
89
  context "using sprintf-ed engine" do
90
90
  let(:config) do
91
91
  {
92
- "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
92
+ "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
93
93
  "engine" => "%{engine_name_field}",
94
- "url" => "http://appsearch:3002"
94
+ "url" => "http://enterprise_search:3002"
95
95
  }
96
96
  end
97
97
 
@@ -134,15 +134,15 @@ describe "indexing against running AppSearch", :integration => true do
134
134
  context "multiple sprintf engines" do
135
135
  let(:config) do
136
136
  {
137
- "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
137
+ "api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
138
138
  "engine" => "%{engine_name_field}",
139
- "url" => "http://appsearch:3002"
139
+ "url" => "http://enterprise_search:3002"
140
140
  }
141
141
  end
142
142
 
143
143
  it "all should be indexed" do
144
- create_engine('testengin1', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
145
- create_engine('testengin2', "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
144
+ create_engine('testengin1', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
145
+ create_engine('testengin2', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
146
146
  events = generate_events(100, 'testengin1')
147
147
  events += generate_events(100, 'testengin2')
148
148
  events.shuffle!
@@ -0,0 +1,83 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/elastic_workplace_search"
4
+ require "logstash/codecs/plain"
5
+ require "logstash/event"
6
+ require "json"
7
+ require "base64"
8
+
9
+ describe "indexing against running Workplace Search", :integration => true do
10
+
11
+ let(:url) { ENV['ENTERPRISE_SEARCH_URL'] }
12
+ let(:auth) { Base64.strict_encode64("#{ENV['ENTERPRISE_SEARCH_USERNAME']}:#{ENV['ENTERPRISE_SEARCH_PASSWORD']}")}
13
+ let(:source) do
14
+ response = Faraday.post(
15
+ "#{url}/ws/org/sources/form_create",
16
+ JSON.dump("service_type" => "custom", "name" => "whatever"),
17
+ "Content-Type" => "application/json",
18
+ "Accept" => "application/json",
19
+ "Authorization" => "Basic #{auth}"
20
+ )
21
+ JSON.load(response.body)
22
+ end
23
+ let(:source_id) { source.fetch("id") }
24
+
25
+ let(:config) do
26
+ {
27
+ "url" => url,
28
+ "source" => source_id,
29
+ "access_token" => source.fetch("accessToken")
30
+ }
31
+ end
32
+
33
+ subject(:workplace_search_output) { LogStash::Outputs::ElasticWorkplaceSearch.new(config) }
34
+
35
+ before(:each) { workplace_search_output.register }
36
+
37
+ describe "single event" do
38
+ let(:event) { LogStash::Event.new("message" => "an event to index") }
39
+
40
+ it "should be indexed" do
41
+ workplace_search_output.multi_receive([event])
42
+
43
+ results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
44
+ attempt_response = execute_search_call
45
+ expect(attempt_response.status).to eq(200)
46
+ parsed_resp = JSON.parse(attempt_response.body)
47
+ expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
48
+ parsed_resp["results"]
49
+ end
50
+ expect(results.first.fetch("message")).to eq "an event to index"
51
+ end
52
+ end
53
+
54
+ describe "multiple events" do
55
+ let(:events) { generate_events(200) } #2 times the slice size used to batch
56
+
57
+ it "all should be indexed" do
58
+ workplace_search_output.multi_receive(events)
59
+ results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
60
+ attempt_response = execute_search_call
61
+ expect(attempt_response.status).to eq(200)
62
+ parsed_resp = JSON.parse(attempt_response.body)
63
+ expect(parsed_resp.dig("meta", "page", "total_results")).to eq(200)
64
+ parsed_resp["results"]
65
+ end
66
+ expect(results.first.fetch("message")).to start_with("an event to index")
67
+ end
68
+ end
69
+
70
+ private
71
+ def execute_search_call
72
+ Faraday.post(
73
+ "#{url}/ws/org/sources/#{source_id}/documents",
74
+ nil,
75
+ "Accept" => "application/json",
76
+ "Authorization" => "Basic #{auth}"
77
+ )
78
+ end
79
+
80
+ def generate_events(num_events)
81
+ (1..num_events).map { |i| LogStash::Event.new("message" => "an event to index #{i}")}
82
+ end
83
+ end
@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/outputs/elastic_app_search"
4
4
  require "logstash/codecs/plain"
5
5
  require "logstash/event"
6
+ require "elastic-app-search"
6
7
 
7
8
  describe LogStash::Outputs::ElasticAppSearch do
8
9
  let(:sample_event) { LogStash::Event.new }
@@ -20,6 +21,19 @@ describe LogStash::Outputs::ElasticAppSearch do
20
21
  it "does not raise an error" do
21
22
  expect { subject.register }.to_not raise_error
22
23
  end
24
+ it "configures the Swiftype client" do
25
+ subject.register
26
+ client = subject.instance_variable_get(:@client)
27
+ expect(client.class).to eq(Elastic::AppSearch::Client)
28
+ end
29
+ end
30
+ context "when path is configured" do
31
+ let(:config) { { "api_key" => api_key, "engine" => engine, "path" => "/v1", "url" => "http://localhost:9300" } }
32
+ it "configures the Swiftype client" do
33
+ subject.register
34
+ client = subject.instance_variable_get(:@client)
35
+ expect(client.class).to eq(Elastic::AppSearch::Client)
36
+ end
23
37
  end
24
38
  context "when host and path is configured" do
25
39
  let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine, "path" => "/v1" } }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-elastic_enterprise_search
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-05-05 00:00:00.000000000 Z
11
+ date: 2022-01-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,48 @@ dependencies:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: 7.8.0
55
+ - !ruby/object:Gem::Dependency
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: 7.16.0
61
+ name: elastic-enterprise-search
62
+ prerelease: false
63
+ type: :runtime
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: 7.16.0
69
+ - !ruby/object:Gem::Dependency
70
+ requirement: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: 0.4.1
75
+ name: elastic-workplace-search
76
+ prerelease: false
77
+ type: :runtime
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: 0.4.1
83
+ - !ruby/object:Gem::Dependency
84
+ requirement: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: '1.0'
89
+ name: logstash-mixin-deprecation_logger_support
90
+ prerelease: false
91
+ type: :runtime
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.0'
55
97
  - !ruby/object:Gem::Dependency
56
98
  requirement: !ruby/object:Gem::Requirement
57
99
  requirements:
@@ -82,8 +124,10 @@ files:
82
124
  - NOTICE.TXT
83
125
  - README.md
84
126
  - lib/logstash/outputs/elastic_app_search.rb
127
+ - lib/logstash/outputs/elastic_workplace_search.rb
85
128
  - logstash-integration-elastic_enterprise_search.gemspec
86
- - spec/integration/outputs/index_spec.rb
129
+ - spec/integration/outputs/elastic_app_search_spec.rb
130
+ - spec/integration/outputs/elastic_workplace_search_spec.rb
87
131
  - spec/unit/outputs/appsearch_spec.rb
88
132
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
89
133
  licenses:
@@ -91,7 +135,7 @@ licenses:
91
135
  metadata:
92
136
  logstash_plugin: 'true'
93
137
  logstash_group: integration
94
- integration_plugins: logstash-output-elastic_app_search
138
+ integration_plugins: logstash-output-elastic_app_search, logstash-output-elastic_workplace_search
95
139
  post_install_message:
96
140
  rdoc_options: []
97
141
  require_paths:
@@ -108,11 +152,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
108
152
  - !ruby/object:Gem::Version
109
153
  version: '0'
110
154
  requirements: []
111
- rubyforge_project:
112
- rubygems_version: 2.6.13
155
+ rubygems_version: 3.1.6
113
156
  signing_key:
114
157
  specification_version: 4
115
158
  summary: Integration with Elastic Enterprise Search - output plugins
116
159
  test_files:
117
- - spec/integration/outputs/index_spec.rb
160
+ - spec/integration/outputs/elastic_app_search_spec.rb
161
+ - spec/integration/outputs/elastic_workplace_search_spec.rb
118
162
  - spec/unit/outputs/appsearch_spec.rb