logstash-integration-elastic_enterprise_search 2.0.0 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -5
- data/DEVELOPER.md +4 -2
- data/README.md +0 -1
- data/lib/logstash/outputs/elastic_app_search.rb +69 -7
- data/lib/logstash/outputs/elastic_workplace_search.rb +106 -0
- data/logstash-integration-elastic_enterprise_search.gemspec +5 -2
- data/spec/integration/outputs/{index_spec.rb → elastic_app_search_spec.rb} +14 -14
- data/spec/integration/outputs/elastic_workplace_search_spec.rb +83 -0
- data/spec/unit/outputs/appsearch_spec.rb +14 -0
- metadata +51 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6276fe38b651d58dcfa68a99d6d6de58f4b7a35d4e5cf640f996cbd7b681a7ad
|
4
|
+
data.tar.gz: 373697b1c10c3c51af094d0415e407033ea1d987d776f4c0e0a83146b998f2cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 898a93fb3d0516b4b6f2a0eb358a29809e581a56ed7c5ca3be21e95f778dd4e63b81e615dbe9bfb96a22cc1439d3f840f0e574a4b56de4e08de3a5eff36ccce9
|
7
|
+
data.tar.gz: 9f4edd07607d41ee54ec36141dd0b009ae8eb5c24364b05a200b419e2efdd800ed57694f10c150e56c1186ee54639fff31361a6c72e9ecff44fb431f251c82b9
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,24 @@
|
|
1
|
-
## 2.0.0
|
2
1
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
## 2.2.0
|
3
|
+
- Feature, switch the connection library to elastic-enterprise-search [#3](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/3)
|
4
|
+
- [DOC] Added required parameters to Workplace Search example snippet and describe little better what's expected in url parameter [#11](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/11)
|
5
|
+
|
6
|
+
## 2.1.2
|
7
|
+
- [DOC] Fix typos in App Search and Workplace Search [#9](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
|
8
|
+
- [DOC] Add links to Elastic App Search and Elastic Workplace Search solution pages [#10](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/9)
|
9
|
+
|
10
|
+
## 2.1.1
|
11
|
+
- [DOC] Added the integration attribute and include statement for the integration plugin header to finalize hooking up integration docs [#8](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/8)
|
12
|
+
- [DOC] Added live link to output-elastic_workplace_search doc [#6](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/6)
|
13
|
+
|
14
|
+
## 2.1.0
|
15
|
+
- Addition of Workplace Search Output plugin [#4](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/4)
|
16
|
+
- [DOC] Added the integration attribute and include statement for the integration plugin header to output-elastic_app_search doc [#5](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/5)
|
17
|
+
|
18
|
+
|
19
|
+
## 2.0.0
|
20
|
+
- Initial release of the Elastic EnterpriseSearch Integration Plugin, which carries the
|
21
|
+
previous AppSearch Output plugin codebase;
|
22
|
+
independent changelogs for previous versions can be found:
|
23
|
+
- [AppSearch Output Plugin @1.2.0](https://github.com/logstash-plugins/logstash-output-elastic_app_search/blob/v1.2.0/CHANGELOG.md)
|
7
24
|
|
data/DEVELOPER.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# logstash-integration-elastic_enterprise_search
|
2
|
-
Elastic Enterprise Search integration for Logstash, including
|
2
|
+
Elastic Enterprise Search integration for Logstash, including App Search and Workplace Search output plugins
|
3
3
|
|
4
4
|
# Dependencies
|
5
|
-
* elastic-
|
5
|
+
* elastic-enterprise-search
|
6
|
+
* elastic-app-search
|
7
|
+
* elastic-workplace-search
|
data/README.md
CHANGED
@@ -97,4 +97,3 @@ Programming is not a required skill. Whatever you've seen about open source and
|
|
97
97
|
It is more important to the community that you are able to contribute.
|
98
98
|
|
99
99
|
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
100
|
-
>>>>>>> 418bfff... Rought output plugin creation
|
@@ -1,32 +1,77 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/outputs/base"
|
3
3
|
require "elastic-app-search"
|
4
|
+
require "elastic-enterprise-search"
|
5
|
+
require 'logstash/plugin_mixins/deprecation_logger_support'
|
4
6
|
|
5
7
|
class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
8
|
+
include LogStash::PluginMixins::DeprecationLoggerSupport
|
9
|
+
|
6
10
|
config_name "elastic_app_search"
|
7
11
|
|
12
|
+
# The name of the search engine you created in App Search, an information
|
13
|
+
# repository that includes the indexed document records.
|
14
|
+
# The `engine` field supports
|
15
|
+
# {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
|
16
|
+
# allow the engine name to be derived from a field value from each event, for
|
17
|
+
# example `engine-%{engine_name}`.
|
18
|
+
#
|
19
|
+
# Invalid engine names cause ingestion to stop until the field value can be resolved into a valid engine name.
|
20
|
+
# This situation can happen if the interpolated field value resolves to a value without a matching engine,
|
21
|
+
# or, if the field is missing from the event and cannot be resolved at all.
|
8
22
|
config :engine, :validate => :string, :required => true
|
23
|
+
|
24
|
+
# The hostname of the App Search API that is associated with your App Search account.
|
25
|
+
# Set this when using the https://www.elastic.co/cloud/app-search-service
|
9
26
|
config :host, :validate => :string
|
27
|
+
|
28
|
+
# The value of the API endpoint in the form of a URL. Note: The value of the of the `path` setting will be will be appended to this URL.
|
29
|
+
# Set this when using the https://www.elastic.co/downloads/app-search
|
10
30
|
config :url, :validate => :string
|
31
|
+
|
32
|
+
# The private API Key with write permissions. Visit the https://app.swiftype.com/as/credentials
|
33
|
+
# in the App Search dashboard to find the key associated with your account.
|
11
34
|
config :api_key, :validate => :password, :required => true
|
35
|
+
|
36
|
+
# Where to move the value from the `@timestamp` field.
|
37
|
+
#
|
38
|
+
# All Logstash events contain a `@timestamp` field.
|
39
|
+
# App Search doesn't support fields starting with `@timestamp`, and
|
40
|
+
# by default, the `@timestamp` field will be deleted.
|
41
|
+
#
|
42
|
+
# To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
|
12
43
|
config :timestamp_destination, :validate => :string
|
44
|
+
|
45
|
+
# The id for app search documents. This can be an interpolated value
|
46
|
+
# like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
|
13
47
|
config :document_id, :validate => :string
|
48
|
+
|
49
|
+
# The path that is appended to the `url` parameter when connecting to a https://www.elastic.co/downloads/app-search
|
14
50
|
config :path, :validate => :string, :default => "/api/as/v1/"
|
15
51
|
|
16
52
|
ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
|
17
53
|
|
18
54
|
public
|
19
55
|
def register
|
56
|
+
@use_old_client = false
|
20
57
|
if @host.nil? && @url.nil?
|
21
58
|
raise ::LogStash::ConfigurationError.new("Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
|
22
59
|
elsif @host && @url
|
23
|
-
raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed) or \"host\" (for SaaS).")
|
60
|
+
raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed ot Elastic Enterprise Search) or \"host\" (for SaaS).")
|
24
61
|
elsif @host && path_is_set? # because path has a default value we need extra work to if the user set it
|
25
62
|
raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
|
26
63
|
elsif @host
|
64
|
+
@deprecation_logger.deprecated("Deprecated service usage, the `host` setting will be removed when Swiftype AppSearch service is shutdown")
|
65
|
+
@use_old_client = true
|
27
66
|
@client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
|
28
67
|
elsif @url
|
29
|
-
|
68
|
+
if path_is_set?
|
69
|
+
@deprecation_logger.deprecated("Deprecated service usage, the `path` setting will be removed when Swiftype AppSearch service is shutdown")
|
70
|
+
@use_old_client = true
|
71
|
+
@client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
|
72
|
+
else
|
73
|
+
@client = Elastic::EnterpriseSearch::AppSearch::Client.new(:host => @url, :http_auth => @api_key.value, :external_url => @url)
|
74
|
+
end
|
30
75
|
end
|
31
76
|
check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
|
32
77
|
rescue => e
|
@@ -45,7 +90,7 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
45
90
|
events.each_slice(100) do |events|
|
46
91
|
batch = format_batch(events)
|
47
92
|
if @logger.trace?
|
48
|
-
@logger.trace("Sending bulk to
|
93
|
+
@logger.trace("Sending bulk to App Search", :size => batch.size, :data => batch.inspect)
|
49
94
|
end
|
50
95
|
index(batch)
|
51
96
|
end
|
@@ -85,11 +130,15 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
85
130
|
if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
|
86
131
|
raise "Cannot resolve engine field name #{@engine} from event"
|
87
132
|
end
|
88
|
-
|
133
|
+
if connected_to_swiftype?
|
134
|
+
response = @client.index_documents(resolved_engine, documents)
|
135
|
+
else
|
136
|
+
response = @client.index_documents(resolved_engine, {:documents => documents})
|
137
|
+
end
|
89
138
|
report(documents, response)
|
90
139
|
rescue => e
|
91
140
|
@logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message,
|
92
|
-
:resolved_engine => resolved_engine)
|
141
|
+
:resolved_engine => resolved_engine, :backtrace => e.backtrace)
|
93
142
|
sleep(1)
|
94
143
|
retry
|
95
144
|
end
|
@@ -98,7 +147,11 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
98
147
|
|
99
148
|
def report(documents, response)
|
100
149
|
documents.each_with_index do |document, i|
|
101
|
-
|
150
|
+
if connected_to_swiftype?
|
151
|
+
errors = response[i]["errors"]
|
152
|
+
else
|
153
|
+
errors = response.body[i]["errors"]
|
154
|
+
end
|
102
155
|
if errors.empty?
|
103
156
|
@logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
|
104
157
|
else
|
@@ -108,10 +161,19 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
108
161
|
end
|
109
162
|
|
110
163
|
def check_connection!
|
111
|
-
|
164
|
+
if connected_to_swiftype?
|
165
|
+
@client.get_engine(@engine)
|
166
|
+
else
|
167
|
+
res = @client.list_engines({:page_size => 1})
|
168
|
+
raise "Received HTTP error code #{res.status}" unless res.status == 200
|
169
|
+
end
|
112
170
|
end
|
113
171
|
|
114
172
|
def path_is_set?
|
115
173
|
original_params.key?("path")
|
116
174
|
end
|
175
|
+
|
176
|
+
def connected_to_swiftype?
|
177
|
+
@use_old_client
|
178
|
+
end
|
117
179
|
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "elastic-workplace-search"
|
4
|
+
|
5
|
+
class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
|
6
|
+
config_name "elastic_workplace_search"
|
7
|
+
|
8
|
+
# The value of the API endpoint in the form of a URL.
|
9
|
+
config :url, :validate => :string, :required => true
|
10
|
+
|
11
|
+
# The ID of the source you created in Workplace Search.
|
12
|
+
# The `source` field supports
|
13
|
+
# {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf format] to
|
14
|
+
# allow the source ID to be derived from a field value from each event, for
|
15
|
+
# example `%{source_id}`.
|
16
|
+
#
|
17
|
+
# Invalid source IDs cause ingestion to stop until the field value can be resolved into a valid source ID.
|
18
|
+
# This situation can happen if the interpolated field value resolves to a value without a matching source,
|
19
|
+
# or, if the field is missing from the event and cannot be resolved at all.
|
20
|
+
config :source, :validate => :string, :required => true
|
21
|
+
|
22
|
+
# The source access token. Visit the source overview page in the Workplace Search dashboard
|
23
|
+
# to find the token associated with your source.
|
24
|
+
config :access_token, :validate => :password, :required => true
|
25
|
+
|
26
|
+
# Where to move the value from the `@timestamp` field.
|
27
|
+
#
|
28
|
+
# All Logstash events contain a `@timestamp` field.
|
29
|
+
# Workplace Search doesn't support fields starting with `@timestamp`, and
|
30
|
+
# by default, the `@timestamp` field will be deleted.
|
31
|
+
#
|
32
|
+
# To keep the timestamp field, set this value to the name of the field where you want `@timestamp` copied.
|
33
|
+
config :timestamp_destination, :validate => :string
|
34
|
+
|
35
|
+
# The id for workplace search documents. This can be an interpolated value
|
36
|
+
# like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
|
37
|
+
config :document_id, :validate => :string
|
38
|
+
|
39
|
+
public
|
40
|
+
def register
|
41
|
+
Elastic::WorkplaceSearch.endpoint = "#{@url.chomp('/')}/api/ws/v1/"
|
42
|
+
@client = Elastic::WorkplaceSearch::Client.new(:access_token => @access_token.value)
|
43
|
+
check_connection!
|
44
|
+
rescue Elastic::WorkplaceSearch::InvalidCredentials
|
45
|
+
raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 401. Please check your credentials")
|
46
|
+
rescue Elastic::WorkplaceSearch::NonExistentRecord
|
47
|
+
raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. Error: 404. Please check if url '#{@url}' is correct and you've created a source with ID '#{@source}'")
|
48
|
+
rescue => e
|
49
|
+
raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. #{e.message}")
|
50
|
+
end
|
51
|
+
|
52
|
+
public
|
53
|
+
def multi_receive(events)
|
54
|
+
# because Workplace Search has a limit of 100 documents per bulk
|
55
|
+
events.each_slice(100) do |events|
|
56
|
+
batch = format_batch(events)
|
57
|
+
if @logger.trace?
|
58
|
+
@logger.trace("Sending bulk to Workplace Search", :size => batch.size, :data => batch.inspect)
|
59
|
+
end
|
60
|
+
index(batch)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
def format_batch(events)
|
66
|
+
events.map do |event|
|
67
|
+
doc = event.to_hash
|
68
|
+
# we need to remove default fields that start with "@"
|
69
|
+
# since Elastic Workplace Search doesn't accept them
|
70
|
+
if @timestamp_destination
|
71
|
+
doc[@timestamp_destination] = doc.delete("@timestamp")
|
72
|
+
else # delete it
|
73
|
+
doc.delete("@timestamp")
|
74
|
+
end
|
75
|
+
if @document_id
|
76
|
+
doc["id"] = event.sprintf(@document_id)
|
77
|
+
end
|
78
|
+
doc.delete("@version")
|
79
|
+
doc
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def index(documents)
|
84
|
+
response = @client.index_documents(@source, documents)
|
85
|
+
report(documents, response)
|
86
|
+
rescue => e
|
87
|
+
@logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message)
|
88
|
+
sleep(1)
|
89
|
+
retry
|
90
|
+
end
|
91
|
+
|
92
|
+
def report(documents, response)
|
93
|
+
documents.each_with_index do |document, i|
|
94
|
+
errors = response["results"][i]["errors"]
|
95
|
+
if errors.empty?
|
96
|
+
@logger.trace? && @logger.trace("Document was indexed with no errors", :document => document)
|
97
|
+
else
|
98
|
+
@logger.warn("Document failed to index. Dropping..", :document => document, :errors => errors.to_a)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def check_connection!
|
104
|
+
@client.list_all_permissions(@source)
|
105
|
+
end
|
106
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-elastic_enterprise_search'
|
3
|
-
s.version = '2.
|
3
|
+
s.version = '2.2.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Integration with Elastic Enterprise Search - output plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
|
@@ -20,12 +20,15 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.metadata = {
|
21
21
|
"logstash_plugin" => "true",
|
22
22
|
"logstash_group" => "integration",
|
23
|
-
"integration_plugins" => "logstash-output-elastic_app_search"
|
23
|
+
"integration_plugins" => "logstash-output-elastic_app_search, logstash-output-elastic_workplace_search"
|
24
24
|
}
|
25
25
|
|
26
26
|
# Gem dependencies
|
27
27
|
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
28
28
|
s.add_runtime_dependency "logstash-codec-plain"
|
29
29
|
s.add_runtime_dependency "elastic-app-search", '~>7.8.0'
|
30
|
+
s.add_runtime_dependency "elastic-enterprise-search", '~>7.16.0'
|
31
|
+
s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
|
32
|
+
s.add_runtime_dependency "logstash-mixin-deprecation_logger_support", '~>1.0'
|
30
33
|
s.add_development_dependency "logstash-devutils"
|
31
34
|
end
|
@@ -5,7 +5,7 @@ require "logstash/codecs/plain"
|
|
5
5
|
require "logstash/event"
|
6
6
|
require "json"
|
7
7
|
|
8
|
-
describe "indexing against running
|
8
|
+
describe "indexing against running App Search", :integration => true do
|
9
9
|
|
10
10
|
let(:engine_name) do
|
11
11
|
(0...10).map { ('a'..'z').to_a[rand(26)] }.join
|
@@ -13,16 +13,16 @@ describe "indexing against running AppSearch", :integration => true do
|
|
13
13
|
|
14
14
|
let(:config) do
|
15
15
|
{
|
16
|
-
"api_key" => ENV['
|
16
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
17
17
|
"engine" => engine_name,
|
18
|
-
"url" => "http://
|
18
|
+
"url" => "http://enterprise_search:3002"
|
19
19
|
}
|
20
20
|
end
|
21
21
|
|
22
22
|
subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
|
23
23
|
|
24
24
|
before(:each) do
|
25
|
-
create_engine(engine_name, "http://
|
25
|
+
create_engine(engine_name, "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
|
26
26
|
end
|
27
27
|
|
28
28
|
private
|
@@ -37,8 +37,8 @@ describe "indexing against running AppSearch", :integration => true do
|
|
37
37
|
describe "search and private keys are configured" do
|
38
38
|
let(:api_key_settings) do
|
39
39
|
{
|
40
|
-
:private => ENV['
|
41
|
-
:search => ENV['
|
40
|
+
:private => ENV['APP_SEARCH_PRIVATE_KEY'],
|
41
|
+
:search => ENV['APP_SEARCH_SEARCH_KEY']
|
42
42
|
}
|
43
43
|
end
|
44
44
|
|
@@ -51,9 +51,9 @@ describe "indexing against running AppSearch", :integration => true do
|
|
51
51
|
describe "register" do
|
52
52
|
let(:config) do
|
53
53
|
{
|
54
|
-
"api_key" => ENV['
|
54
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
55
55
|
"engine" => "%{engine_name_field}",
|
56
|
-
"url" => "http://
|
56
|
+
"url" => "http://enterprise_search:3002"
|
57
57
|
}
|
58
58
|
end
|
59
59
|
|
@@ -89,9 +89,9 @@ describe "indexing against running AppSearch", :integration => true do
|
|
89
89
|
context "using sprintf-ed engine" do
|
90
90
|
let(:config) do
|
91
91
|
{
|
92
|
-
"api_key" => ENV['
|
92
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
93
93
|
"engine" => "%{engine_name_field}",
|
94
|
-
"url" => "http://
|
94
|
+
"url" => "http://enterprise_search:3002"
|
95
95
|
}
|
96
96
|
end
|
97
97
|
|
@@ -134,15 +134,15 @@ describe "indexing against running AppSearch", :integration => true do
|
|
134
134
|
context "multiple sprintf engines" do
|
135
135
|
let(:config) do
|
136
136
|
{
|
137
|
-
"api_key" => ENV['
|
137
|
+
"api_key" => ENV['APP_SEARCH_PRIVATE_KEY'],
|
138
138
|
"engine" => "%{engine_name_field}",
|
139
|
-
"url" => "http://
|
139
|
+
"url" => "http://enterprise_search:3002"
|
140
140
|
}
|
141
141
|
end
|
142
142
|
|
143
143
|
it "all should be indexed" do
|
144
|
-
create_engine('testengin1', "http://
|
145
|
-
create_engine('testengin2', "http://
|
144
|
+
create_engine('testengin1', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
|
145
|
+
create_engine('testengin2', "http://enterprise_search:3002", ENV['APP_SEARCH_PRIVATE_KEY'])
|
146
146
|
events = generate_events(100, 'testengin1')
|
147
147
|
events += generate_events(100, 'testengin2')
|
148
148
|
events.shuffle!
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/elastic_workplace_search"
|
4
|
+
require "logstash/codecs/plain"
|
5
|
+
require "logstash/event"
|
6
|
+
require "json"
|
7
|
+
require "base64"
|
8
|
+
|
9
|
+
describe "indexing against running Workplace Search", :integration => true do
|
10
|
+
|
11
|
+
let(:url) { ENV['ENTERPRISE_SEARCH_URL'] }
|
12
|
+
let(:auth) { Base64.strict_encode64("#{ENV['ENTERPRISE_SEARCH_USERNAME']}:#{ENV['ENTERPRISE_SEARCH_PASSWORD']}")}
|
13
|
+
let(:source) do
|
14
|
+
response = Faraday.post(
|
15
|
+
"#{url}/ws/org/sources/form_create",
|
16
|
+
JSON.dump("service_type" => "custom", "name" => "whatever"),
|
17
|
+
"Content-Type" => "application/json",
|
18
|
+
"Accept" => "application/json",
|
19
|
+
"Authorization" => "Basic #{auth}"
|
20
|
+
)
|
21
|
+
JSON.load(response.body)
|
22
|
+
end
|
23
|
+
let(:source_id) { source.fetch("id") }
|
24
|
+
|
25
|
+
let(:config) do
|
26
|
+
{
|
27
|
+
"url" => url,
|
28
|
+
"source" => source_id,
|
29
|
+
"access_token" => source.fetch("accessToken")
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
subject(:workplace_search_output) { LogStash::Outputs::ElasticWorkplaceSearch.new(config) }
|
34
|
+
|
35
|
+
before(:each) { workplace_search_output.register }
|
36
|
+
|
37
|
+
describe "single event" do
|
38
|
+
let(:event) { LogStash::Event.new("message" => "an event to index") }
|
39
|
+
|
40
|
+
it "should be indexed" do
|
41
|
+
workplace_search_output.multi_receive([event])
|
42
|
+
|
43
|
+
results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
|
44
|
+
attempt_response = execute_search_call
|
45
|
+
expect(attempt_response.status).to eq(200)
|
46
|
+
parsed_resp = JSON.parse(attempt_response.body)
|
47
|
+
expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
|
48
|
+
parsed_resp["results"]
|
49
|
+
end
|
50
|
+
expect(results.first.fetch("message")).to eq "an event to index"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe "multiple events" do
|
55
|
+
let(:events) { generate_events(200) } #2 times the slice size used to batch
|
56
|
+
|
57
|
+
it "all should be indexed" do
|
58
|
+
workplace_search_output.multi_receive(events)
|
59
|
+
results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
|
60
|
+
attempt_response = execute_search_call
|
61
|
+
expect(attempt_response.status).to eq(200)
|
62
|
+
parsed_resp = JSON.parse(attempt_response.body)
|
63
|
+
expect(parsed_resp.dig("meta", "page", "total_results")).to eq(200)
|
64
|
+
parsed_resp["results"]
|
65
|
+
end
|
66
|
+
expect(results.first.fetch("message")).to start_with("an event to index")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
def execute_search_call
|
72
|
+
Faraday.post(
|
73
|
+
"#{url}/ws/org/sources/#{source_id}/documents",
|
74
|
+
nil,
|
75
|
+
"Accept" => "application/json",
|
76
|
+
"Authorization" => "Basic #{auth}"
|
77
|
+
)
|
78
|
+
end
|
79
|
+
|
80
|
+
def generate_events(num_events)
|
81
|
+
(1..num_events).map { |i| LogStash::Event.new("message" => "an event to index #{i}")}
|
82
|
+
end
|
83
|
+
end
|
@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
|
|
3
3
|
require "logstash/outputs/elastic_app_search"
|
4
4
|
require "logstash/codecs/plain"
|
5
5
|
require "logstash/event"
|
6
|
+
require "elastic-app-search"
|
6
7
|
|
7
8
|
describe LogStash::Outputs::ElasticAppSearch do
|
8
9
|
let(:sample_event) { LogStash::Event.new }
|
@@ -20,6 +21,19 @@ describe LogStash::Outputs::ElasticAppSearch do
|
|
20
21
|
it "does not raise an error" do
|
21
22
|
expect { subject.register }.to_not raise_error
|
22
23
|
end
|
24
|
+
it "configures the Swiftype client" do
|
25
|
+
subject.register
|
26
|
+
client = subject.instance_variable_get(:@client)
|
27
|
+
expect(client.class).to eq(Elastic::AppSearch::Client)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
context "when path is configured" do
|
31
|
+
let(:config) { { "api_key" => api_key, "engine" => engine, "path" => "/v1", "url" => "http://localhost:9300" } }
|
32
|
+
it "configures the Swiftype client" do
|
33
|
+
subject.register
|
34
|
+
client = subject.instance_variable_get(:@client)
|
35
|
+
expect(client.class).to eq(Elastic::AppSearch::Client)
|
36
|
+
end
|
23
37
|
end
|
24
38
|
context "when host and path is configured" do
|
25
39
|
let(:config) { { "host" => host, "api_key" => api_key, "engine" => engine, "path" => "/v1" } }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-integration-elastic_enterprise_search
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,6 +52,48 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 7.8.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 7.16.0
|
61
|
+
name: elastic-enterprise-search
|
62
|
+
prerelease: false
|
63
|
+
type: :runtime
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 7.16.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 0.4.1
|
75
|
+
name: elastic-workplace-search
|
76
|
+
prerelease: false
|
77
|
+
type: :runtime
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.4.1
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '1.0'
|
89
|
+
name: logstash-mixin-deprecation_logger_support
|
90
|
+
prerelease: false
|
91
|
+
type: :runtime
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.0'
|
55
97
|
- !ruby/object:Gem::Dependency
|
56
98
|
requirement: !ruby/object:Gem::Requirement
|
57
99
|
requirements:
|
@@ -82,8 +124,10 @@ files:
|
|
82
124
|
- NOTICE.TXT
|
83
125
|
- README.md
|
84
126
|
- lib/logstash/outputs/elastic_app_search.rb
|
127
|
+
- lib/logstash/outputs/elastic_workplace_search.rb
|
85
128
|
- logstash-integration-elastic_enterprise_search.gemspec
|
86
|
-
- spec/integration/outputs/
|
129
|
+
- spec/integration/outputs/elastic_app_search_spec.rb
|
130
|
+
- spec/integration/outputs/elastic_workplace_search_spec.rb
|
87
131
|
- spec/unit/outputs/appsearch_spec.rb
|
88
132
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
89
133
|
licenses:
|
@@ -91,7 +135,7 @@ licenses:
|
|
91
135
|
metadata:
|
92
136
|
logstash_plugin: 'true'
|
93
137
|
logstash_group: integration
|
94
|
-
integration_plugins: logstash-output-elastic_app_search
|
138
|
+
integration_plugins: logstash-output-elastic_app_search, logstash-output-elastic_workplace_search
|
95
139
|
post_install_message:
|
96
140
|
rdoc_options: []
|
97
141
|
require_paths:
|
@@ -108,11 +152,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
108
152
|
- !ruby/object:Gem::Version
|
109
153
|
version: '0'
|
110
154
|
requirements: []
|
111
|
-
|
112
|
-
rubygems_version: 2.6.13
|
155
|
+
rubygems_version: 3.1.6
|
113
156
|
signing_key:
|
114
157
|
specification_version: 4
|
115
158
|
summary: Integration with Elastic Enterprise Search - output plugins
|
116
159
|
test_files:
|
117
|
-
- spec/integration/outputs/
|
160
|
+
- spec/integration/outputs/elastic_app_search_spec.rb
|
161
|
+
- spec/integration/outputs/elastic_workplace_search_spec.rb
|
118
162
|
- spec/unit/outputs/appsearch_spec.rb
|