logstash-integration-elastic_enterprise_search 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -5
- data/DEVELOPER.md +4 -2
- data/README.md +0 -1
- data/lib/logstash/outputs/elastic_app_search.rb +69 -7
- data/lib/logstash/outputs/elastic_workplace_search.rb +106 -0
- data/logstash-integration-elastic_enterprise_search.gemspec +5 -2
- data/spec/integration/outputs/{index_spec.rb → elastic_app_search_spec.rb} +14 -14
- data/spec/integration/outputs/elastic_workplace_search_spec.rb +83 -0
- data/spec/unit/outputs/appsearch_spec.rb +14 -0
- metadata +51 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6276fe38b651d58dcfa68a99d6d6de58f4b7a35d4e5cf640f996cbd7b681a7ad
|
4
|
+
data.tar.gz: 373697b1c10c3c51af094d0415e407033ea1d987d776f4c0e0a83146b998f2cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 898a93fb3d0516b4b6f2a0eb358a29809e581a56ed7c5ca3be21e95f778dd4e63b81e615dbe9bfb96a22cc1439d3f840f0e574a4b56de4e08de3a5eff36ccce9
|
7
|
+
data.tar.gz: 9f4edd07607d41ee54ec36141dd0b009ae8eb5c24364b05a200b419e2efdd800ed57694f10c150e56c1186ee54639fff31361a6c72e9ecff44fb431f251c82b9
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,24 @@
|
|
1
|
-
## 2.0.0
|
2
1
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
## 2.2.0
|
3
|
+
- Feature, switch the connection library to elastic-enterprise-search [#3](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/3)
|
4
|
+
- [DOC] Added required parameters to Workplace Search example snippet and describe little better what's expected in url parameter [#11](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/11)
|
5
|
+
|
6
|
+
## 2.1.2
|
7
|
+
- [DOC] Fix typos in App Search and Workplace Search [#9](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
|
8
|
+
- [DOC] Add links to Elastic App Search and Elastic Workplace Search solution pages [#10](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
|
9
|
+
|
10
|
+
## 2.1.1
|
11
|
+
- [DOC] Added the integration attribute and include statement for the integration plugin header to finalize hooking up integration docs [#8](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/8)
|
12
|
+
- [DOC] Added live link to output-elastic_workplace_search doc [#6](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/6)
|
13
|
+
|
14
|
+
## 2.1.0
|
15
|
+
- Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
|
16
|
+
- [DOC] Added the integration attribute and include statement for the integration plugin header to output-elastic_app_search doc [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
|
17
|
+
|
18
|
+
|
19
|
+
## 2.0.0
|
20
|
+
- Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
|
21
|
+
previous AppSearch Output plugin codebase;
|
22
|
+
independent changelogs for previous versions can be found:
|
23
|
+
- [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
|
7
24
|
|
data/DEVELOPER.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# logstash-integration-elastic_enterprise_search
|
2
|
-
Elastic Enterprise Search integration for Logstash, including
|
2
|
+
Elastic Enterprise Search integration for Logstash, including App Search and Workplace Search output plugins
|
3
3
|
|
4
4
|
# Dependencies
|
5
|
-
* elastic-
|
5
|
+
* elastic-enterprise-search
|
6
|
+
* elastic-app-search
|
7
|
+
* elastic-workplace-search
|
data/README.md
CHANGED
@@ -97,4 +97,3 @@ Programming is not a required skill. Whatever you've seen about open source and
|
|
97
97
|
It is more important to the community that you are able to contribute.
|
98
98
|
|
99
99
|
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
100
|
-
>>>>>>> 418bfff... Rought output plugin creation
|
@@ -1,32 +1,77 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/outputs/base"
|
3
3
|
require "elastic-app-search"
|
4
|
+
require "elastic-enterprise-search"
|
5
|
+
require 'logstash/plugin_mixins/deprecation_logger_support'
|
4
6
|
|
5
7
|
class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
8
|
+
include LogStash::PluginMixins::DeprecationLoggerSupport
|
9
|
+
|
6
10
|
config_name "elastic_app_search"
|
7
11
|
|
12
|
+
# The name of the search engine you created in App Search, an information
|
13
|
+
# repository that includes the indexed document records.
|
14
|
+
# The `engine` field supports
|
15
|
+
# {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
|
16
|
+
# allow the engine name to be derived from a field value from each event, for
|
17
|
+
# example `engine-%{engine_name}`.
|
18
|
+
#
|
19
|
+
# Invalid engine names cause ingestion to stop until the field value can be resolved into a valid engine name.
|
20
|
+
# This situation can happen if the interpolated field value resolves to a value without a matching engine,
|
21
|
+
# or, if the field is missing from the event and cannot be resolved at all.
|
8
22
|
config :engine, :validate => :string, :required => true
|
23
|
+
|
24
|
+
# The hostname of the App Search API that is associated with your App Search account.
|
25
|
+
# Set this when using the https://www.elastic.co/cloud/app-search-service
|
9
26
|
config :host, :validate => :string
|
27
|
+
|
28
|
+
# The value of the API endpoint in the form of a URL. Note: The value of the of the `path` setting will be will be appended to this URL.
|
29
|
+
# Set this when using the https://www.elastic.co/downloads/app-search
|
10
30
|
config :url, :validate => :string
|
31
|
+
|
32
|
+
# The private API Key with write permissions. Visit the https://app.swiftype.com/as/credentials
|
33
|
+
# in the App Search dashboard to find the key associated with your account.
|
11
34
|
config :api_key, :validate => :password, :required => true
|
35
|
+
|
36
|
+
# Where to move the value from the `@timestamp` field.
|
37
|
+
#
|
38
|
+
# All Logstash events contain a `@timestamp` field.
|
39
|
+
# App Search doesn't support fields starting with `@timestamp`, and
|
40
|
+
# by default, the `@timestamp` field will be deleted.
|
41
|
+
#
|
42
|
+
# To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
|
12
43
|
config :timestamp_destination, :validate => :string
|
44
|
+
|
45
|
+
# The id for app search documents. This can be an interpolated value
|
46
|
+
# like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
|
13
47
|
config :document_id, :validate => :string
|
48
|
+
|
49
|
+
# The path that is appended to the `url` parameter when connecting to a https://www.elastic.co/downloads/app-search
|
14
50
|
config :path, :validate => :string, :default => "/api/as/v1/"
|
15
51
|
|
16
52
|
ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
|
17
53
|
|
18
54
|
public
|
19
55
|
def register
|
56
|
+
@use_old_client = false
|
20
57
|
if @host.nil? && @url.nil?
|
21
58
|
raise ::LogStash::ConfigurationError.new("Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
|
22
59
|
elsif @host && @url
|
23
|
-
raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
|
60
|
+
raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed ot Elastic Enterprise Search) or \"host\" (for SaaS).")
|
24
61
|
elsif @host && path_is_set? # because path has a default value we need extra work to if the user set it
|
25
62
|
raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
|
26
63
|
elsif @host
|
64
|
+
@deprecation_logger.deprecated("Deprecated service usage, the `host` setting will be removed when Swiftype AppSearch service is shutdown")
|
65
|
+
@use_old_client = true
|
27
66
|
@client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
|
28
67
|
elsif @url
|
29
|
-
|
68
|
+
if path_is_set?
|
69
|
+
@deprecation_logger.deprecated("Deprecated service usage, the `path` setting will be removed when Swiftype AppSearch service is shutdown")
|
70
|
+
@use_old_client = true
|
71
|
+
@client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
|
72
|
+
else
|
73
|
+
@client = Elastic::EnterpriseSearch::AppSearch::Client.new(:host => @url, :http_auth => @api_key.value, :external_url => @url)
|
74
|
+
end
|
30
75
|
end
|
31
76
|
check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
|
32
77
|
rescue => e
|
@@ -45,7 +90,7 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
45
90
|
events.each_slice(100) do |events|
|
46
91
|
batch = format_batch(events)
|
47
92
|
if @logger.trace?
|
48
|
-
@logger.trace("Sending bulk to
|
93
|
+
@logger.trace("Sending bulk to App Search", :size => batch.size, :data => batch.inspect)
|
49
94
|
end
|
50
95
|
index(batch)
|
51
96
|
end
|
@@ -85,11 +130,15 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
85
130
|
if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
|
86
131
|
raise "Cannot resolve engine field name #{@engine} from event"
|
87
132
|
end
|
88
|
-
|
133
|
+
if connected_to_swiftype?
|
134
|
+
response = @client.index_documents(resolved_engine, documents)
|
135
|
+
else
|
136
|
+
response = @client.index_documents(resolved_engine, {:documents => documents})
|
137
|
+
end
|
89
138
|
report(documents, response)
|
90
139
|
rescue => e
|
91
140
|
@logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
|
92
|
-
:resolved_engine => resolved_engine)
|
141
|
+
:resolved_engine => resolved_engine, :backtrace => e.backtrace)
|
93
142
|
sleep(1)
|
94
143
|
retry
|
95
144
|
end
|
@@ -98,7 +147,11 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
98
147
|
|
99
148
|
def report(documents, response)
|
100
149
|
documents.each_with_index do |document, i|
|
101
|
-
|
150
|
+
if connected_to_swiftype?
|
151
|
+
errors = response[i]["errors"]
|
152
|
+
else
|
153
|
+
errors = response.body[i]["errors"]
|
154
|
+
end
|
102
155
|
if errors.empty?
|
103
156
|
@logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
|
104
157
|
else
|
@@ -108,10 +161,19 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
108
161
|
end
|
109
162
|
|
110
163
|
def check_connection!
|
111
|
-
|
164
|
+
if connected_to_swiftype?
|
165
|
+
@client.get_engine(@engine)
|
166
|
+
else
|
167
|
+
res = @client.list_engines({:page_size => 1})
|
168
|
+
raise "Received HTTP error code #{res.status}" unless res.status == 200
|
169
|
+
end
|
112
170
|
end
|
113
171
|
|
114
172
|
def path_is_set?
|
115
173
|
original_params.key?("path")
|
116
174
|
end
|
175
|
+
|
176
|
+
def connected_to_swiftype?
|
177
|
+
@use_old_client
|
178
|
+
end
|
117
179
|
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "elastic-workplace-search"
|
4
|
+
|
5
|
+
class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
|
6
|
+
config_name "elastic_workplace_search"
|
7
|
+
|
8
|
+
# The value of the API endpoint in the form of a URL.
|
9
|
+
config :url, :validate => :string, :required => true
|
10
|
+
|
11
|
+
# The ID of the source you created in Workplace Search.
|
12
|
+
# The `source` field supports
|
13
|
+
# {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
|
14
|
+
# allow the source ID to be derived from a field value from each event, for
|
15
|
+
# example `%{source_id}`.
|
16
|
+
#
|
17
|
+
# Invalid source IDs cause ingestion to stop until the field value can be resolved into a valid source ID.
|
18
|
+
# This situation can happen if the interpolated field value resolves to a value without a matching source,
|
19
|
+
# or, if the field is missing from the event and cannot be resolved at all.
|
20
|
+
config :source, :validate => :string, :required => true
|
21
|
+
|
22
|
+
# The source access token. Visit the source overview page in the Workplace Search dashboard
|
23
|
+
# to find the token associated with your source.
|
24
|
+
config :access_token, :validate => :password, :required => true
|
25
|
+
|
26
|
+
# Where to move the value from the `@timestamp` field.
|
27
|
+
#
|
28
|
+
# All Logstash events contain a `@timestamp` field.
|
29
|
+
# Workplace Search doesn't support fields starting with `@timestamp`, and
|
30
|
+
# by default, the `@timestamp` field will be deleted.
|
31
|
+
#
|
32
|
+
# To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
|
33
|
+
config :timestamp_destination, :validate => :string
|
34
|
+
|
35
|
+
# The id for workplace search documents. This can be an interpolated value
|
36
|
+
# like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
|
37
|
+
config :document_id, :validate => :string
|
38
|
+
|
39
|
+
public
|
40
|
+
def register
|
41
|
+
Elastic::WorkplaceSearch.endpoint = "#{@url.chomp('/')}/api/ws/v1/"
|
42
|
+
@client = Elastic::WorkplaceSearch::Client.new(:access_token => @access_token.value)
|
43
|
+
check_connection!
|
44
|
+
rescue Elastic::WorkplaceSearch::InvalidCredentials
|
45
|
+
raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 401. Please check your credentials")
|
46
|
+
rescue Elastic::WorkplaceSearch::NonExistentRecord
|
47
|
+
raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 404. Please check if url '#{@url}' is correct and you've created a source with ID '#{@source}'")
|
48
|
+
rescue => e
|
49
|
+
raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. #{e.message}")
|
50
|
+
end
|
51
|
+
|
52
|
+
public
|
53
|
+
def multi_receive(events)
|
54
|
+
# because Workplace Search has a limit of 100 documents per bulk
|
55
|
+
events.each_slice(100) do |events|
|
56
|
+
batch = format_batch(events)
|
57
|
+
if @logger.trace?
|
58
|
+
@logger.trace("Sending bulk to Workplace Search", :size => batch.size, :data => batch.inspect)
|
59
|
+
end
|
60
|
+
index(batch)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
def format_batch(events)
|
66
|
+
events.map do |event|
|
67
|
+
doc = event.to_hash
|
68
|
+
# we need to remove default fields that start with "@"
|
69
|
+
# since Elastic Workplace Search doesn't accept them
|
70
|
+
if @timestamp_destination
|
71
|
+
doc[@timestamp_destination] = doc.delete("@timestamp")
|
72
|
+
else # delete it
|
73
|
+
doc.delete("@timestamp")
|
74
|
+
end
|
75
|
+
if @document_id
|
76
|
+
doc["id"] = event.sprintf(@document_id)
|
77
|
+
end
|
78
|
+
doc.delete("@version")
|
79
|
+
doc
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def index(documents)
|
84
|
+
response = @client.index_documents(@source, documents)
|
85
|
+
report(documents, response)
|
86
|
+
rescue => e
|
87
|
+
@logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message)
|
88
|
+
sleep(1)
|
89
|
+
retry
|
90
|
+
end
|
91
|
+
|
92
|
+
def report(documents, response)
|
93
|
+
documents.each_with_index do |document, i|
|
94
|
+
errors = response["results"][i]["errors"]
|
95
|
+
if errors.empty?
|
96
|
+
@logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
|
97
|
+
else
|
98
|
+
@logger.warn("Document failed to index. Dropping..", :document => document, :errors => errors.to_a)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def check_connection!
|
104
|
+
@client.list_all_permissions(@source)
|
105
|
+
end
|
106
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-elastic_enterprise_search'
|
3
|
-
s.version = '2.
|
3
|
+
s.version = '2.2.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Integration with Elastic Enterprise Search - output plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
|
@@ -20,12 +20,15 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.metadata = {
|
21
21
|
"logstash_plugin" => "true",
|
22
22
|
"logstash_group" => "integration",
|
23
|
-
"integration_plugins" => "logstash-output-elastic_app_search"
|
23
|
+
"integration_plugins" => "logstash-output-elastic_app_search, logstash-output-elastic_workplace_search"
|
24
24
|
}
|
25
25
|
|
26
26
|
# Gem dependencies
|
27
27
|
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
28
28
|
s.add_runtime_dependency "logstash-codec-plain"
|
29
29
|
s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
|
30
|
+
s.add_runtime_dependency "elastic-enterprise-search", '~>7.16.0'
|
31
|
+
s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
|
32
|
+
s.add_runtime_dependency "logstash-mixin-deprecation_logger_support", '~>1.0'
|
30
33
|
s.add_development_dependency "logstash-devutils"
|
31
34
|
end
|
@@ -5,7 +5,7 @@ require "logstash/codecs/plain"
|
|
5
5
|
require "logstash/event"
|
6
6
|
require "json"
|
7
7
|
|
8
|
-
describe "indexing against running
|
8
|
+
describe "indexing against running App Search", :integration => true do
|
9
9
|
|
10
10
|
let(:engine_name) do
|
11
11
|
(0...10).map { ('a'..'z').to_a[rand(26)] }.join
|
@@ -13,16 +13,16 @@ describe "indexing against running AppSearch", :integration => true do
|
|
13
13
|
|
14
14
|
let(:config) do
|
15
15
|
{
|
16
|
-
"api_key" => ENV['
|
16
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
17
17
|
"engine" => engine_name,
|
18
|
-
"url" => "http://
|
18
|
+
"url" => "http://enterprise_search:3002"
|
19
19
|
}
|
20
20
|
end
|
21
21
|
|
22
22
|
subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
|
23
23
|
|
24
24
|
before(:each) do
|
25
|
-
create_engine(engine_name, "http://
|
25
|
+
create_engine(engine_name, "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
|
26
26
|
end
|
27
27
|
|
28
28
|
private
|
@@ -37,8 +37,8 @@ describe "indexing against running AppSearch", :integration => true do
|
|
37
37
|
describe "search and private keys are configured" do
|
38
38
|
let(:api_key_settings) do
|
39
39
|
{
|
40
|
-
:private => ENV['
|
41
|
-
:search => ENV['
|
40
|
+
:private => ENV['APP_SEARCH_PRIVATE_KEY'],
|
41
|
+
:search => ENV['APP_SEARCH_SEARCH_KEY']
|
42
42
|
}
|
43
43
|
end
|
44
44
|
|
@@ -51,9 +51,9 @@ describe "indexing against running AppSearch", :integration => true do
|
|
51
51
|
describe "register" do
|
52
52
|
let(:config) do
|
53
53
|
{
|
54
|
-
"api_key" => ENV['
|
54
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
55
55
|
"engine" => "%{engine_name_field}",
|
56
|
-
"url" => "http://
|
56
|
+
"url" => "http://enterprise_search:3002"
|
57
57
|
}
|
58
58
|
end
|
59
59
|
|
@@ -89,9 +89,9 @@ describe "indexing against running AppSearch", :integration => true do
|
|
89
89
|
context "using sprintf-ed engine" do
|
90
90
|
let(:config) do
|
91
91
|
{
|
92
|
-
"api_key" => ENV['
|
92
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
93
93
|
"engine" => "%{engine_name_field}",
|
94
|
-
"url" => "http://
|
94
|
+
"url" => "http://enterprise_search:3002"
|
95
95
|
}
|
96
96
|
end
|
97
97
|
|
@@ -134,15 +134,15 @@ describe "indexing against running AppSearch", :integration => true do
|
|
134
134
|
context "multiple sprintf engines" do
|
135
135
|
let(:config) do
|
136
136
|
{
|
137
|
-
"api_key" => ENV['
|
137
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
138
138
|
"engine" => "%{engine_name_field}",
|
139
|
-
"url" => "http://
|
139
|
+
"url" => "http://enterprise_search:3002"
|
140
140
|
}
|
141
141
|
end
|
142
142
|
|
143
143
|
it "all should be indexed" do
|
144
|
-
create_engine('testengin1', "http://
|
145
|
-
create_engine('testengin2', "http://
|
144
|
+
create_engine('testengin1', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
|
145
|
+
create_engine('testengin2', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
|
146
146
|
events = generate_events(100, 'testengin1')
|
147
147
|
events += generate_events(100, 'testengin2')
|
148
148
|
events.shuffle!
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/elastic_workplace_search"
|
4
|
+
require "logstash/codecs/plain"
|
5
|
+
require "logstash/event"
|
6
|
+
require "json"
|
7
|
+
require "base64"
|
8
|
+
|
9
|
+
describe "indexing against running Workplace Search", :integration => true do
|
10
|
+
|
11
|
+
let(:url) { ENV['ENTERPRISE_SEARCH_URL'] }
|
12
|
+
let(:auth) { Base64.strict_encode64("#{ENV['ENTERPRISE_SEARCH_USERNAME']}:#{ENV['ENTERPRISE_SEARCH_PASSWORD']}")}
|
13
|
+
let(:source) do
|
14
|
+
response = Faraday.post(
|
15
|
+
"#{url}/ws/org/sources/form_create",
|
16
|
+
JSON.dump("service_type" => "custom", "name" => "whatever"),
|
17
|
+
"Content-Type" => "application/json",
|
18
|
+
"Accept" => "application/json",
|
19
|
+
"Authorization" => "Basic #{auth}"
|
20
|
+
)
|
21
|
+
JSON.load(response.body)
|
22
|
+
end
|
23
|
+
let(:source_id) { source.fetch("id") }
|
24
|
+
|
25
|
+
let(:config) do
|
26
|
+
{
|
27
|
+
"url" => url,
|
28
|
+
"source" => source_id,
|
29
|
+
"access_token" => source.fetch("accessToken")
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
subject(:workplace_search_output) { LogStash::Outputs::ElasticWorkplaceSearch.new(config) }
|
34
|
+
|
35
|
+
before(:each) { workplace_search_output.register }
|
36
|
+
|
37
|
+
describe "single event" do
|
38
|
+
let(:event) { LogStash::Event.new("message" => "an event to index") }
|
39
|
+
|
40
|
+
it "should be indexed" do
|
41
|
+
workplace_search_output.multi_receive([event])
|
42
|
+
|
43
|
+
results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
|
44
|
+
attempt_response = execute_search_call
|
45
|
+
expect(attempt_response.status).to eq(200)
|
46
|
+
parsed_resp = JSON.parse(attempt_response.body)
|
47
|
+
expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
|
48
|
+
parsed_resp["results"]
|
49
|
+
end
|
50
|
+
expect(results.first.fetch("message")).to eq "an event to index"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe "multiple events" do
|
55
|
+
let(:events) { generate_events(200) } #2 times the slice size used to batch
|
56
|
+
|
57
|
+
it "all should be indexed" do
|
58
|
+
workplace_search_output.multi_receive(events)
|
59
|
+
results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
|
60
|
+
attempt_response = execute_search_call
|
61
|
+
expect(attempt_response.status).to eq(200)
|
62
|
+
parsed_resp = JSON.parse(attempt_response.body)
|
63
|
+
expect(parsed_resp.dig("meta", "page", "total_results")).to eq(200)
|
64
|
+
parsed_resp["results"]
|
65
|
+
end
|
66
|
+
expect(results.first.fetch("message")).to start_with("an event to index")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
def execute_search_call
|
72
|
+
Faraday.post(
|
73
|
+
"#{url}/ws/org/sources/#{source_id}/documents",
|
74
|
+
nil,
|
75
|
+
"Accept" => "application/json",
|
76
|
+
"Authorization" => "Basic #{auth}"
|
77
|
+
)
|
78
|
+
end
|
79
|
+
|
80
|
+
def generate_events(num_events)
|
81
|
+
(1..num_events).map { |i| LogStash::Event.new("message" => "an event to index #{i}")}
|
82
|
+
end
|
83
|
+
end
|
@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
|
|
3
3
|
require "logstash/outputs/elastic_app_search"
|
4
4
|
require "logstash/codecs/plain"
|
5
5
|
require "logstash/event"
|
6
|
+
require "elastic-app-search"
|
6
7
|
|
7
8
|
describe LogStash::Outputs::ElasticAppSearch do
|
8
9
|
let(:sample_event) { LogStash::Event.new }
|
@@ -20,6 +21,19 @@ describe LogStash::Outputs::ElasticAppSearch do
|
|
20
21
|
it "does not raise an error" do
|
21
22
|
expect { subject.register }.to_not raise_error
|
22
23
|
end
|
24
|
+
it "configures the Swiftype client" do
|
25
|
+
subject.register
|
26
|
+
client = subject.instance_variable_get(:@client)
|
27
|
+
expect(client.class).to eq(Elastic::AppSearch::Client)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
context "when path is configured" do
|
31
|
+
let(:config) { { "api_key" => api_key, "engine" => engine, "path" => "/v1", "url" => "http://localhost:9300" } }
|
32
|
+
it "configures the Swiftype client" do
|
33
|
+
subject.register
|
34
|
+
client = subject.instance_variable_get(:@client)
|
35
|
+
expect(client.class).to eq(Elastic::AppSearch::Client)
|
36
|
+
end
|
23
37
|
end
|
24
38
|
context "when host and path is configured" do
|
25
39
|
let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine, "path" => "/v1" } }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-integration-elastic_enterprise_search
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,6 +52,48 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 7.8.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 7.16.0
|
61
|
+
name: elastic-enterprise-search
|
62
|
+
prerelease: false
|
63
|
+
type: :runtime
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 7.16.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 0.4.1
|
75
|
+
name: elastic-workplace-search
|
76
|
+
prerelease: false
|
77
|
+
type: :runtime
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.4.1
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '1.0'
|
89
|
+
name: logstash-mixin-deprecation_logger_support
|
90
|
+
prerelease: false
|
91
|
+
type: :runtime
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.0'
|
55
97
|
- !ruby/object:Gem::Dependency
|
56
98
|
requirement: !ruby/object:Gem::Requirement
|
57
99
|
requirements:
|
@@ -82,8 +124,10 @@ files:
|
|
82
124
|
- NOTICE.TXT
|
83
125
|
- README.md
|
84
126
|
- lib/logstash/outputs/elastic_app_search.rb
|
127
|
+
- lib/logstash/outputs/elastic_workplace_search.rb
|
85
128
|
- logstash-integration-elastic_enterprise_search.gemspec
|
86
|
-
- spec/integration/outputs/
|
129
|
+
- spec/integration/outputs/elastic_app_search_spec.rb
|
130
|
+
- spec/integration/outputs/elastic_workplace_search_spec.rb
|
87
131
|
- spec/unit/outputs/appsearch_spec.rb
|
88
132
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
89
133
|
licenses:
|
@@ -91,7 +135,7 @@ licenses:
|
|
91
135
|
metadata:
|
92
136
|
logstash_plugin: 'true'
|
93
137
|
logstash_group: integration
|
94
|
-
integration_plugins: logstash-output-elastic_app_search
|
138
|
+
integration_plugins: logstash-output-elastic_app_search, logstash-output-elastic_workplace_search
|
95
139
|
post_install_message:
|
96
140
|
rdoc_options: []
|
97
141
|
require_paths:
|
@@ -108,11 +152,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
108
152
|
- !ruby/object:Gem::Version
|
109
153
|
version: '0'
|
110
154
|
requirements: []
|
111
|
-
|
112
|
-
rubygems_version: 2.6.13
|
155
|
+
rubygems_version: 3.1.6
|
113
156
|
signing_key:
|
114
157
|
specification_version: 4
|
115
158
|
summary: Integration with Elastic Enterprise Search - output plugins
|
116
159
|
test_files:
|
117
|
-
- spec/integration/outputs/
|
160
|
+
- spec/integration/outputs/elastic_app_search_spec.rb
|
161
|
+
- spec/integration/outputs/elastic_workplace_search_spec.rb
|
118
162
|
- spec/unit/outputs/appsearch_spec.rb
|