logstash-integration-elastic_enterprise_search 2.2.1 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +10 -0
- data/lib/logstash/outputs/elastic_app_search.rb +45 -81
- data/lib/logstash/outputs/elastic_workplace_search.rb +67 -37
- data/lib/logstash/plugin_mixins/enterprise_search/client.rb +35 -0
- data/lib/logstash/plugin_mixins/enterprise_search/manticore_transport.rb +84 -0
- data/lib/logstash/plugin_mixins/enterprise_search/ssl_configs.rb +32 -0
- data/logstash-integration-elastic_enterprise_search.gemspec +3 -4
- data/spec/fixtures/certificates/generate.sh +10 -0
- data/spec/fixtures/certificates/root_ca.crt +34 -0
- data/spec/fixtures/certificates/root_ca.key +52 -0
- data/spec/fixtures/certificates/root_keystore.jks +0 -0
- data/spec/fixtures/certificates/root_untrusted_ca.crt +34 -0
- data/spec/fixtures/certificates/root_untrusted_keystore.jks +0 -0
- data/spec/fixtures/certificates/root_untrusted_keystore.p12 +0 -0
- data/spec/integration/outputs/elastic_app_search_spec.rb +133 -105
- data/spec/integration/outputs/elastic_workplace_search_spec.rb +162 -95
- data/spec/unit/outputs/client_spec.rb +26 -0
- data/spec/unit/outputs/elastic_app_search_spec.rb +117 -0
- data/spec/unit/outputs/elastic_workplace_search_spec.rb +117 -0
- data/spec/unit/outputs/manticore_transport_spec.rb +124 -0
- metadata +50 -35
- data/spec/unit/outputs/appsearch_spec.rb +0 -64
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b06802185d7a0e45e13ac9a3f5a952901e7fdd186d78c50dd5f38c7db18f2c41
|
4
|
+
data.tar.gz: 263045ba6b2947b74a418b663de0dcdf63f67475bb99ac4f1252f5bfd6cec4b1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d294f1592ce25cb470ddd7746e095974b005722a022c6eb8faad76f30ac8d895c02b9afa1be2d9715e5d25fd4c1eb8a206e22b15d1ed84f9f27fb0e21d4806bc
|
7
|
+
data.tar.gz: 99fe5e6638912b63c977cca07f85456e00a627dee50475d5e5fa628a49fe062c0f77b7c1994c208391d1675240df755e1f3de4d6e32e9160ea48408b0a318572
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 3.0.0
|
2
|
+
- Bumped Enterprise Search clients to version `>= 7.16`, `< 9` [#18](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/18)
|
3
|
+
- Added support to SSL configurations (`ssl_certificate_authorities`, `ssl_truststore_path`, `ssl_truststore_password`, `ssl_truststore_type`, `ssl_verification_mode`, `ssl_supported_protocols` and `ssl_cipher_suites`)
|
4
|
+
- [BREAKING] Swiftype endpoints are no longer supported for both plugins App Search and Workplace Search
|
5
|
+
- The App Search deprecated options `host` and `path` were removed
|
6
|
+
- Fixed the sprintf format support for the Workplace Search `source` configuration
|
7
|
+
|
1
8
|
## 2.2.1
|
2
9
|
- Fix, change implementation of connectivity check method to be compatible with version `v8.0+` of Workplace Search [#16](https://github.com/logstash-plugins/logstash-integration-elastic_enterprise_search/pull/16)
|
3
10
|
|
data/README.md
CHANGED
@@ -88,6 +88,16 @@ bin/plugin install --no-verify
|
|
88
88
|
```
|
89
89
|
- Start Logstash and proceed to test the plugin
|
90
90
|
|
91
|
+
### Thoubleshooting integration test failures
|
92
|
+
Integration tests uses some certificates fixtures. These security artifacts has 1 year expiration, so time to time
|
93
|
+
they would trigger errors due to bad certificate.
|
94
|
+
To regenerate use:
|
95
|
+
```sh
|
96
|
+
> cd spec/fixture/certificates
|
97
|
+
> ./generate.sh
|
98
|
+
```
|
99
|
+
Re-run locally the integration tests (with Docker scripts) and if it's green again create a PR to update.
|
100
|
+
|
91
101
|
## Contributing
|
92
102
|
|
93
103
|
All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
|
@@ -1,13 +1,13 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require 'logstash/plugin_mixins/deprecation_logger_support'
|
2
|
+
require 'logstash/outputs/base'
|
3
|
+
require 'logstash/plugin_mixins/enterprise_search/ssl_configs'
|
4
|
+
require 'logstash/plugin_mixins/enterprise_search/client'
|
6
5
|
|
7
6
|
class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
8
|
-
include LogStash::PluginMixins::DeprecationLoggerSupport
|
9
7
|
|
10
|
-
|
8
|
+
include LogStash::PluginMixins::EnterpriseSearch::SSLConfigs
|
9
|
+
|
10
|
+
config_name 'elastic_app_search'
|
11
11
|
|
12
12
|
# The name of the search engine you created in App Search, an information
|
13
13
|
# repository that includes the indexed document records.
|
@@ -21,16 +21,11 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
21
21
|
# or, if the field is missing from the event and cannot be resolved at all.
|
22
22
|
config :engine, :validate => :string, :required => true
|
23
23
|
|
24
|
-
# The
|
25
|
-
|
26
|
-
config :host, :validate => :string
|
27
|
-
|
28
|
-
# The value of the API endpoint in the form of a URL. Note: The value of the of the `path` setting will be will be appended to this URL.
|
29
|
-
# Set this when using the https://www.elastic.co/downloads/app-search
|
30
|
-
config :url, :validate => :string
|
24
|
+
# The value of the API endpoint in the form of a URL.
|
25
|
+
config :url, :validate => :string, :required => true, :default => Elastic::EnterpriseSearch::Utils::DEFAULT_HOST
|
31
26
|
|
32
|
-
# The private API Key with write permissions.
|
33
|
-
#
|
27
|
+
# The private API Key with write permissions.
|
28
|
+
# https://www.elastic.co/guide/en/app-search/current/authentication.html#authentication-api-keys
|
34
29
|
config :api_key, :validate => :password, :required => true
|
35
30
|
|
36
31
|
# Where to move the value from the `@timestamp` field.
|
@@ -46,57 +41,42 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
46
41
|
# like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
|
47
42
|
config :document_id, :validate => :string
|
48
43
|
|
49
|
-
|
50
|
-
config :path, :validate => :string, :default => "/api/as/v1/"
|
51
|
-
|
52
|
-
ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/
|
44
|
+
ENGINE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/.freeze
|
53
45
|
|
54
|
-
public
|
55
46
|
def register
|
56
|
-
@
|
57
|
-
|
58
|
-
|
59
|
-
elsif @host && @url
|
60
|
-
raise ::LogStash::ConfigurationError.new("Both \"url\" or \"host\" can't be set simultaneously. Please specify either \"url\" (for self-managed ot Elastic Enterprise Search) or \"host\" (for SaaS).")
|
61
|
-
elsif @host && path_is_set? # because path has a default value we need extra work to if the user set it
|
62
|
-
raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
|
63
|
-
elsif @host
|
64
|
-
@deprecation_logger.deprecated("Deprecated service usage, the `host` setting will be removed when Swiftype AppSearch service is shutdown")
|
65
|
-
@use_old_client = true
|
66
|
-
@client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
|
67
|
-
elsif @url
|
68
|
-
if path_is_set?
|
69
|
-
@deprecation_logger.deprecated("Deprecated service usage, the `path` setting will be removed when Swiftype AppSearch service is shutdown")
|
70
|
-
@use_old_client = true
|
71
|
-
@client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
|
72
|
-
else
|
73
|
-
@client = Elastic::EnterpriseSearch::AppSearch::Client.new(:host => @url, :http_auth => @api_key.value, :external_url => @url)
|
74
|
-
end
|
75
|
-
end
|
76
|
-
check_connection! unless @engine =~ ENGINE_WITH_SPRINTF_REGEX
|
47
|
+
@retry_disabled = false
|
48
|
+
@client = LogStash::PluginMixins::EnterpriseSearch::AppSearch::Client.new(client_options, params: params)
|
49
|
+
check_connection!
|
77
50
|
rescue => e
|
78
51
|
if e.message =~ /401/
|
79
|
-
raise ::LogStash::ConfigurationError
|
52
|
+
raise ::LogStash::ConfigurationError, "Failed to connect to App Search. Please check your credentials. Error: #{e.message}"
|
80
53
|
elsif e.message =~ /404/
|
81
|
-
raise ::LogStash::ConfigurationError
|
54
|
+
raise ::LogStash::ConfigurationError, "Failed to connect to App Search. Please check if url '#{@url}' is correct and you've created an engine with name '#{@engine}'. Error: #{e.message}"
|
82
55
|
else
|
83
|
-
raise ::LogStash::ConfigurationError
|
56
|
+
raise ::LogStash::ConfigurationError, "Failed to connect to App Search. Error: #{e.message}"
|
84
57
|
end
|
85
58
|
end
|
86
59
|
|
87
|
-
public
|
88
60
|
def multi_receive(events)
|
89
61
|
# because App Search has a limit of 100 documents per bulk
|
90
62
|
events.each_slice(100) do |events|
|
91
63
|
batch = format_batch(events)
|
92
64
|
if @logger.trace?
|
93
|
-
@logger.trace(
|
65
|
+
@logger.trace('Sending bulk to App Search', :size => batch.size, :data => batch.inspect)
|
94
66
|
end
|
95
67
|
index(batch)
|
96
68
|
end
|
97
69
|
end
|
98
70
|
|
99
71
|
private
|
72
|
+
|
73
|
+
def client_options
|
74
|
+
options = { :host => @url, :http_auth => @api_key.value }
|
75
|
+
options[:logger] = @logger if @logger.debug?
|
76
|
+
options[:tracer] = @logger if @logger.trace?
|
77
|
+
options
|
78
|
+
end
|
79
|
+
|
100
80
|
def format_batch(events)
|
101
81
|
docs_for_engine = {}
|
102
82
|
events.each do |event|
|
@@ -104,18 +84,18 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
104
84
|
# we need to remove default fields that start with "@"
|
105
85
|
# since Elastic App Search doesn't accept them
|
106
86
|
if @timestamp_destination
|
107
|
-
doc[@timestamp_destination] = doc.delete(
|
87
|
+
doc[@timestamp_destination] = doc.delete('@timestamp')
|
108
88
|
else # delete it
|
109
|
-
doc.delete(
|
89
|
+
doc.delete('@timestamp')
|
110
90
|
end
|
111
91
|
if @document_id
|
112
|
-
doc[
|
92
|
+
doc['id'] = event.sprintf(@document_id)
|
113
93
|
end
|
114
|
-
doc.delete(
|
94
|
+
doc.delete('@version')
|
115
95
|
resolved_engine = event.sprintf(@engine)
|
116
96
|
unless docs_for_engine[resolved_engine]
|
117
97
|
if @logger.debug?
|
118
|
-
@logger.debug(
|
98
|
+
@logger.debug('Creating new engine segment in batch to send', :resolved_engine => resolved_engine)
|
119
99
|
end
|
120
100
|
docs_for_engine[resolved_engine] = []
|
121
101
|
end
|
@@ -124,21 +104,21 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
124
104
|
docs_for_engine
|
125
105
|
end
|
126
106
|
|
127
|
-
def index(
|
128
|
-
|
107
|
+
def index(docs_partitioned_by_engine)
|
108
|
+
docs_partitioned_by_engine.each do |resolved_engine, documents|
|
129
109
|
begin
|
130
110
|
if resolved_engine =~ ENGINE_WITH_SPRINTF_REGEX || resolved_engine =~ /^\s*$/
|
131
111
|
raise "Cannot resolve engine field name #{@engine} from event"
|
132
112
|
end
|
133
|
-
|
134
|
-
|
135
|
-
else
|
136
|
-
response = @client.index_documents(resolved_engine, {:documents => documents})
|
137
|
-
end
|
113
|
+
|
114
|
+
response = @client.index_documents(resolved_engine, { :documents => documents })
|
138
115
|
report(documents, response)
|
139
116
|
rescue => e
|
140
|
-
@logger.error(
|
141
|
-
:resolved_engine => resolved_engine, :backtrace => e.backtrace)
|
117
|
+
@logger.error('Failed to execute index operation.', :exception => e.class, :reason => e.message,
|
118
|
+
:resolved_engine => resolved_engine, :backtrace => e.backtrace, :retry => !@retry_disabled)
|
119
|
+
|
120
|
+
raise e if @retry_disabled
|
121
|
+
|
142
122
|
sleep(1)
|
143
123
|
retry
|
144
124
|
end
|
@@ -147,33 +127,17 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
|
|
147
127
|
|
148
128
|
def report(documents, response)
|
149
129
|
documents.each_with_index do |document, i|
|
150
|
-
|
151
|
-
errors = response[i]["errors"]
|
152
|
-
else
|
153
|
-
errors = response.body[i]["errors"]
|
154
|
-
end
|
130
|
+
errors = response.body[i]['errors']
|
155
131
|
if errors.empty?
|
156
|
-
@logger.trace? && @logger.trace(
|
132
|
+
@logger.trace? && @logger.trace('Document was indexed with no errors', :document => document)
|
157
133
|
else
|
158
|
-
@logger.warn(
|
134
|
+
@logger.warn('Document failed to index. Dropping..', :document => document, :errors => errors.to_a)
|
159
135
|
end
|
160
136
|
end
|
161
137
|
end
|
162
138
|
|
163
139
|
def check_connection!
|
164
|
-
|
165
|
-
|
166
|
-
else
|
167
|
-
res = @client.list_engines({:page_size => 1})
|
168
|
-
raise "Received HTTP error code #{res.status}" unless res.status == 200
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
|
-
def path_is_set?
|
173
|
-
original_params.key?("path")
|
174
|
-
end
|
175
|
-
|
176
|
-
def connected_to_swiftype?
|
177
|
-
@use_old_client
|
140
|
+
res = @client.list_engines({ 'page[size]': 1 })
|
141
|
+
raise "Received HTTP error code #{res.status}" unless res.status == 200
|
178
142
|
end
|
179
143
|
end
|
@@ -1,12 +1,16 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require
|
3
|
-
require
|
2
|
+
require 'logstash/outputs/base'
|
3
|
+
require 'logstash/plugin_mixins/enterprise_search/client'
|
4
|
+
require 'logstash/plugin_mixins/enterprise_search/ssl_configs'
|
4
5
|
|
5
6
|
class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
|
6
|
-
|
7
|
+
|
8
|
+
include LogStash::PluginMixins::EnterpriseSearch::SSLConfigs
|
9
|
+
|
10
|
+
config_name 'elastic_workplace_search'
|
7
11
|
|
8
12
|
# The value of the API endpoint in the form of a URL.
|
9
|
-
config :url, :validate => :string, :required => true
|
13
|
+
config :url, :validate => :string, :required => true, :default => Elastic::EnterpriseSearch::Utils::DEFAULT_HOST
|
10
14
|
|
11
15
|
# The ID of the source you created in Workplace Search.
|
12
16
|
# The `source` field supports
|
@@ -36,72 +40,98 @@ class LogStash::Outputs::ElasticWorkplaceSearch < LogStash::Outputs::Base
|
|
36
40
|
# like `myapp-%{sequence_id}`. Reusing ids will cause documents to be rewritten.
|
37
41
|
config :document_id, :validate => :string
|
38
42
|
|
39
|
-
|
43
|
+
SOURCE_WITH_SPRINTF_REGEX = /^.*%\{.+\}.*$/.freeze
|
44
|
+
|
40
45
|
def register
|
41
|
-
|
42
|
-
@client =
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
rescue => e
|
49
|
-
raise ::LogStash::ConfigurationError.new("Failed to connect to Workplace Search. #{e.message}")
|
46
|
+
@retry_disabled = false
|
47
|
+
@client = LogStash::PluginMixins::EnterpriseSearch::WorkplaceSearch::Client.new(client_options, params: params)
|
48
|
+
begin
|
49
|
+
check_connection!
|
50
|
+
rescue => e
|
51
|
+
raise ::LogStash::ConfigurationError, "Failed to connect to Workplace Search. Error: #{e.message}"
|
52
|
+
end
|
50
53
|
end
|
51
54
|
|
52
|
-
public
|
53
55
|
def multi_receive(events)
|
54
56
|
# because Workplace Search has a limit of 100 documents per bulk
|
55
57
|
events.each_slice(100) do |events|
|
56
58
|
batch = format_batch(events)
|
57
|
-
if @logger.trace?
|
58
|
-
@logger.trace("Sending bulk to Workplace Search", :size => batch.size, :data => batch.inspect)
|
59
|
-
end
|
59
|
+
@logger.trace('Sending bulk to Workplace Search', :size => batch.size, :data => batch.inspect) if @logger.trace?
|
60
60
|
index(batch)
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
64
|
private
|
65
|
+
|
66
|
+
def client_options
|
67
|
+
options = { :host => @url, :http_auth => @access_token.value }
|
68
|
+
options[:logger] = @logger if @logger.debug?
|
69
|
+
options[:tracer] = @logger if @logger.trace?
|
70
|
+
options
|
71
|
+
end
|
72
|
+
|
65
73
|
def format_batch(events)
|
74
|
+
docs_per_source = {}
|
66
75
|
events.map do |event|
|
67
76
|
doc = event.to_hash
|
68
77
|
# we need to remove default fields that start with "@"
|
69
78
|
# since Elastic Workplace Search doesn't accept them
|
70
79
|
if @timestamp_destination
|
71
|
-
doc[@timestamp_destination] = doc.delete(
|
80
|
+
doc[@timestamp_destination] = doc.delete('@timestamp')
|
72
81
|
else # delete it
|
73
|
-
doc.delete(
|
82
|
+
doc.delete('@timestamp')
|
83
|
+
end
|
84
|
+
|
85
|
+
doc['id'] = event.sprintf(@document_id) if @document_id
|
86
|
+
doc.delete('@version')
|
87
|
+
|
88
|
+
resolved_source = event.sprintf(@source)
|
89
|
+
unless docs_per_source[resolved_source]
|
90
|
+
@logger.debug('Creating new source segment in batch to send', resolved_source: resolved_source) if @logger.debug?
|
91
|
+
docs_per_source[resolved_source] = []
|
74
92
|
end
|
75
|
-
|
76
|
-
|
93
|
+
docs_per_source[resolved_source] << doc
|
94
|
+
end
|
95
|
+
|
96
|
+
docs_per_source
|
97
|
+
end
|
98
|
+
|
99
|
+
def index(docs_partitioned_by_source)
|
100
|
+
docs_partitioned_by_source.each do |resolved_source, documents|
|
101
|
+
begin
|
102
|
+
raise "Cannot resolve source field name #{@source} from event" unless resolved?(resolved_source)
|
103
|
+
|
104
|
+
response = @client.index_documents(resolved_source, { :documents => documents })
|
105
|
+
report(documents, response)
|
106
|
+
rescue => e
|
107
|
+
@logger.error('Failed to execute index operation.', :exception => e.class, :reason => e.message,
|
108
|
+
:resolved_source => resolved_source, :backtrace => e.backtrace, :retry => !@retry_disabled)
|
109
|
+
|
110
|
+
raise e if @retry_disabled
|
111
|
+
|
112
|
+
sleep(1)
|
113
|
+
retry
|
77
114
|
end
|
78
|
-
doc.delete("@version")
|
79
|
-
doc
|
80
115
|
end
|
81
116
|
end
|
82
117
|
|
83
|
-
def
|
84
|
-
|
85
|
-
report(documents, response)
|
86
|
-
rescue => e
|
87
|
-
@logger.error("Failed to execute index operation. Retrying..", :exception => e.class, :reason => e.message)
|
88
|
-
sleep(1)
|
89
|
-
retry
|
118
|
+
def resolved?(source)
|
119
|
+
!(source =~ SOURCE_WITH_SPRINTF_REGEX) && !(source =~ /^\s*$/)
|
90
120
|
end
|
91
121
|
|
92
122
|
def report(documents, response)
|
93
123
|
documents.each_with_index do |document, i|
|
94
|
-
errors = response
|
95
|
-
if errors
|
96
|
-
@logger.trace? && @logger.trace(
|
124
|
+
errors = response.body&.dig('results', i, 'errors')
|
125
|
+
if errors&.empty?
|
126
|
+
@logger.trace? && @logger.trace('Document was indexed with no errors', :document => document)
|
97
127
|
else
|
98
|
-
@logger.warn(
|
128
|
+
@logger.warn('Document failed to index. Dropping..', :document => document, :errors => errors.to_a)
|
99
129
|
end
|
100
130
|
end
|
101
131
|
end
|
102
132
|
|
103
133
|
def check_connection!
|
104
|
-
|
105
|
-
|
134
|
+
res = @client.list_content_sources({ 'page[size]': 1 })
|
135
|
+
raise "Received HTTP error code #{res.status}" unless res.status == 200
|
106
136
|
end
|
107
137
|
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'elastic-enterprise-search'
|
2
|
+
require 'logstash/plugin_mixins/enterprise_search/manticore_transport'
|
3
|
+
|
4
|
+
module LogStash::PluginMixins::EnterpriseSearch
|
5
|
+
|
6
|
+
module AppSearch
|
7
|
+
# App Search client for Enterprise Search.
|
8
|
+
# This client extends Elastic::EnterpriseSearch::AppSearch::Client but overrides #transport to use Manticore.
|
9
|
+
class Client < Elastic::EnterpriseSearch::AppSearch::Client
|
10
|
+
attr_reader :params
|
11
|
+
|
12
|
+
include LogStash::PluginMixins::EnterpriseSearch::ManticoreTransport
|
13
|
+
|
14
|
+
def initialize(options, params: {})
|
15
|
+
@params = params
|
16
|
+
super options
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
module WorkplaceSearch
|
22
|
+
# Workplace Search client for Enterprise Search.
|
23
|
+
# This client extends Elastic::EnterpriseSearch::WorkplaceSearch::Client but overrides #transport to use Manticore.
|
24
|
+
class Client < Elastic::EnterpriseSearch::WorkplaceSearch::Client
|
25
|
+
attr_reader :params
|
26
|
+
|
27
|
+
include LogStash::PluginMixins::EnterpriseSearch::ManticoreTransport
|
28
|
+
|
29
|
+
def initialize(options, params: {})
|
30
|
+
@params = params
|
31
|
+
super options
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
module LogStash::PluginMixins::EnterpriseSearch
|
2
|
+
# This module is meant to be included in EnterpriseSearch::Clients subclasses and overrides
|
3
|
+
# the default #transport method, changing the HTTP client to Manticore.
|
4
|
+
# It also provides common Manticore configurations such as :ssl.
|
5
|
+
module ManticoreTransport
|
6
|
+
def self.included(base)
|
7
|
+
if eps_version_7?
|
8
|
+
require 'elasticsearch/transport/transport/http/manticore'
|
9
|
+
else
|
10
|
+
require 'elastic/transport/transport/http/manticore'
|
11
|
+
end
|
12
|
+
|
13
|
+
raise ArgumentError, "`#{base}` must inherit Elastic::EnterpriseSearch::Client" unless base < Elastic::EnterpriseSearch::Client
|
14
|
+
raise ArgumentError, "`#{base}` must respond to :params" unless base.instance_methods.include? :params
|
15
|
+
end
|
16
|
+
|
17
|
+
# overrides Elastic::EnterpriseSearch::Client#transport
|
18
|
+
def transport
|
19
|
+
@options[:transport] || transport_klass.new(
|
20
|
+
host: host,
|
21
|
+
log: log,
|
22
|
+
logger: logger,
|
23
|
+
request_timeout: overall_timeout,
|
24
|
+
adapter: @options[:adapter],
|
25
|
+
transport_options: {
|
26
|
+
request: { open_timeout: open_timeout }
|
27
|
+
},
|
28
|
+
transport_class: manticore_transport_klass,
|
29
|
+
ssl: build_ssl_config,
|
30
|
+
enable_meta_header: @options[:enable_meta_header] || true,
|
31
|
+
trace: !@options[:tracer].nil?,
|
32
|
+
tracer: @options[:tracer]
|
33
|
+
)
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.eps_version_7?
|
37
|
+
Elastic::EnterpriseSearch::VERSION.start_with?('7')
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def manticore_transport_klass
|
43
|
+
ManticoreTransport.eps_version_7? ? Elasticsearch::Transport::Transport::HTTP::Manticore : Elastic::Transport::Transport::HTTP::Manticore
|
44
|
+
end
|
45
|
+
|
46
|
+
def transport_klass
|
47
|
+
case Elasticsearch::Transport::VERSION
|
48
|
+
when /7\.1[123]/
|
49
|
+
Elasticsearch::Client
|
50
|
+
else
|
51
|
+
Elasticsearch::Transport::Client
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def build_transport_options
|
56
|
+
{ request: { open_timeout: open_timeout } }
|
57
|
+
end
|
58
|
+
|
59
|
+
def build_ssl_config
|
60
|
+
ssl_certificate_authorities, ssl_truststore_path = params.values_at('ssl_certificate_authorities', 'ssl_truststore_path')
|
61
|
+
if ssl_certificate_authorities && ssl_truststore_path
|
62
|
+
raise ::LogStash::ConfigurationError, 'Use either "ssl_certificate_authorities" or "ssl_truststore_path" when configuring the CA certificate'
|
63
|
+
end
|
64
|
+
|
65
|
+
ssl_options = {}
|
66
|
+
if ssl_certificate_authorities&.any?
|
67
|
+
raise ::LogStash::ConfigurationError, 'Multiple values on "ssl_certificate_authorities" are not supported by this plugin' if ssl_certificate_authorities.size > 1
|
68
|
+
|
69
|
+
ssl_options[:ca_file] = ssl_certificate_authorities.first
|
70
|
+
end
|
71
|
+
|
72
|
+
if ssl_truststore_path
|
73
|
+
ssl_options[:truststore] = ssl_truststore_path
|
74
|
+
ssl_options[:truststore_type] = params['ssl_truststore_type'] if params.include?('ssl_truststore_type')
|
75
|
+
ssl_options[:truststore_password] = params['ssl_truststore_password'].value if params.include?('ssl_truststore_password')
|
76
|
+
end
|
77
|
+
|
78
|
+
ssl_options[:verify] = params['ssl_verification_mode'] == 'full' ? :strict : :disable
|
79
|
+
ssl_options[:cipher_suites] = params['ssl_cipher_suites'] if params.include?('ssl_cipher_suites')
|
80
|
+
ssl_options[:protocols] = params['ssl_supported_protocols'] if params['ssl_supported_protocols']&.any?
|
81
|
+
ssl_options
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module LogStash::PluginMixins::EnterpriseSearch
|
2
|
+
# This module defines common SSL options that can be reused by the app and workplace search plugins.
|
3
|
+
module SSLConfigs
|
4
|
+
def self.included(base)
|
5
|
+
# SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary
|
6
|
+
base.config :ssl_certificate_authorities, :validate => :path, :list => true
|
7
|
+
|
8
|
+
# The JKS truststore to validate the server's certificate.
|
9
|
+
# Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
|
10
|
+
base.config :ssl_truststore_path, :validate => :path
|
11
|
+
|
12
|
+
# Set the truststore password
|
13
|
+
base.config :ssl_truststore_password, :validate => :password
|
14
|
+
|
15
|
+
# The format of the truststore file. It must be either jks or pkcs12
|
16
|
+
base.config :ssl_truststore_type, :validate => %w[pkcs12 jks]
|
17
|
+
|
18
|
+
# Options to verify the server's certificate.
|
19
|
+
# "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
|
20
|
+
# chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
|
21
|
+
# "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
|
22
|
+
base.config :ssl_verification_mode, :validate => %w[full none], :default => 'full'
|
23
|
+
|
24
|
+
# Supported protocols with versions.
|
25
|
+
base.config :ssl_supported_protocols, :validate => %w[TLSv1.1 TLSv1.2 TLSv1.3], :default => [], :list => true
|
26
|
+
|
27
|
+
# The list of cipher suites to use, listed by priorities.
|
28
|
+
# Supported cipher suites vary depending on which version of Java is used.
|
29
|
+
base.config :ssl_cipher_suites, :validate => :string, :list => true
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-elastic_enterprise_search'
|
3
|
-
s.version = '
|
3
|
+
s.version = '3.0.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Integration with Elastic Enterprise Search - output plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+
|
@@ -24,11 +24,10 @@ Gem::Specification.new do |s|
|
|
24
24
|
}
|
25
25
|
|
26
26
|
# Gem dependencies
|
27
|
+
s.add_runtime_dependency "manticore", '~> 0.8'
|
27
28
|
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
28
29
|
s.add_runtime_dependency "logstash-codec-plain"
|
29
|
-
s.add_runtime_dependency
|
30
|
-
s.add_runtime_dependency "elastic-enterprise-search", '~>7.16.0'
|
31
|
-
s.add_runtime_dependency "elastic-workplace-search", '~>0.4.1'
|
30
|
+
s.add_runtime_dependency 'elastic-enterprise-search', '>= 7.16', '< 9'
|
32
31
|
s.add_runtime_dependency "logstash-mixin-deprecation_logger_support", '~>1.0'
|
33
32
|
s.add_development_dependency "logstash-devutils"
|
34
33
|
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# Warning: do not use the certificates produced by this tool in production. This is for testing purposes only
|
2
|
+
|
3
|
+
openssl genrsa 4096 | openssl pkcs8 -topk8 -nocrypt -out root_ca.key
|
4
|
+
openssl req -sha256 -x509 -newkey rsa:4096 -nodes -key root_ca.key -sha256 -days 365 -out root_ca.crt -subj "/C=ES/ST=The Internet/L=The Internet/O=Logstash CA/OU=Logstash/CN=enterprise_search"
|
5
|
+
openssl req -sha256 -x509 -newkey rsa:4096 -nodes -key root_ca.key -sha256 -days 365 -out root_untrusted_ca.crt -subj "/C=ES/ST=The Darknet/L=The Darknet/O=Logstash CA/OU=Logstash/CN=127.0.0.1"
|
6
|
+
openssl pkcs12 -export -in root_ca.crt -inkey root_ca.key -out root_keystore.p12 -password pass:changeme -name ent-search
|
7
|
+
keytool -importkeystore -srckeystore root_keystore.p12 -destkeystore root_keystore.jks -srcstoretype PKCS12 -deststoretype jks -srcstorepass changeme -deststorepass changeme -srcalias ent-search -destalias ent-search -srckeypass changeme -destkeypass changeme
|
8
|
+
|
9
|
+
rm -rf root_keystore.p12
|
10
|
+
rm -rf *.csr
|
@@ -0,0 +1,34 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIF4zCCA8ugAwIBAgIUQ9JjwQESSVBfFxIEw50tni49yaUwDQYJKoZIhvcNAQEL
|
3
|
+
BQAwgYAxCzAJBgNVBAYTAkVTMRUwEwYDVQQIDAxUaGUgSW50ZXJuZXQxFTATBgNV
|
4
|
+
BAcMDFRoZSBJbnRlcm5ldDEUMBIGA1UECgwLTG9nc3Rhc2ggQ0ExETAPBgNVBAsM
|
5
|
+
CExvZ3N0YXNoMRowGAYDVQQDDBFlbnRlcnByaXNlX3NlYXJjaDAeFw0yMzEwMTYx
|
6
|
+
NDUwNDZaFw0yNDEwMTUxNDUwNDZaMIGAMQswCQYDVQQGEwJFUzEVMBMGA1UECAwM
|
7
|
+
VGhlIEludGVybmV0MRUwEwYDVQQHDAxUaGUgSW50ZXJuZXQxFDASBgNVBAoMC0xv
|
8
|
+
Z3N0YXNoIENBMREwDwYDVQQLDAhMb2dzdGFzaDEaMBgGA1UEAwwRZW50ZXJwcmlz
|
9
|
+
ZV9zZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDnyKWaGkRr
|
10
|
+
3N8z7lG7cD49FZlk8p26AVG213xKZaj4VlfJfcQZQ8AoGXzOFI5ow6qhnChglT7t
|
11
|
+
yngZU16TZdY0Iosh1ZFvt7S3N00tamYQdAH1IUQPvygHPZvs66Ik3G/+NOFeGVQK
|
12
|
+
YFkUiuxUgGJ13Q+lFMRSshDay0rf8ekRnFE73roHBveo8Pr7TWDB/eacphQe+RPE
|
13
|
+
VB4VYILqv/J+PByzYxidFMdhY5rlMYyRk/SOsOqUcXVLw94yo2D42OIcNMSkpdpx
|
14
|
+
xTH42iInWeGgyWkMCDZV9aUWNcXdZ+zRLO0IZD3LxRJ0c/QSFjwrfIVoSkNQyMq7
|
15
|
+
b9HPimsc3Ud+YsRQiYuRsMIeAVFNrIZxbbC61o///hWvM2loTai8uOhdxj72l2pC
|
16
|
+
q3VzI+i3rkX0x0/oFx00h686batF+bPHwBUuJpzf+KMJPCiV5Z916TnjHA1OrtBd
|
17
|
+
6WH+ckFmL+6TwjzLk9mg9iYWVFnky9+mG6c2svgjn05KA++2r2rVsXQPx8DwppgD
|
18
|
+
dEk570ciu0EhP94oZlw0smtf/uhp/wVym9Z1IOjo3i19DKeF+DzPTD2ylSi/43ub
|
19
|
+
ICeKGG70ZT3eP2Nes58z/GXhxHFYHLUJcg6KjvaPWiv6KciGWwtR/XuT2aoShlpp
|
20
|
+
sZv6u5ckPWxYvhZK0+07yxyoCrLgGGO1XQIDAQABo1MwUTAdBgNVHQ4EFgQU+pka
|
21
|
+
cSbOMgRJk0Gi4ChZrWo8vrMwHwYDVR0jBBgwFoAU+pkacSbOMgRJk0Gi4ChZrWo8
|
22
|
+
vrMwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAeQHpkts7hzod
|
23
|
+
NzawMbawu6zPx3d6hP1OaUqU4sWa5OHu0eLebDLn4HQ9q6FJjuFREGHjktfKJOaf
|
24
|
+
lpflQrCioHcmicQ9lTHqm7SyjokteYCbQMJOLpn41TzHUVlufxqTgPGf3QjjcrEh
|
25
|
+
SaplxNFZxFXJO383aZfELfLojLtE4bZK8olx7l2mnPSVbhe/TJzpIoFvICOfw/H4
|
26
|
+
pdE4SuLJGblaMBV57guRziI+PBqDulUeq92/ZWE5kDO32itgxEObNzbehiUcKNS8
|
27
|
+
LeCiniGXvvGAUaWnekueMJeYm65cJZzp5uLULSW5aL4mCBuyJskpmHb3iYZvthEy
|
28
|
+
vJwNZZc10/i7ogfUUMQUtmvYTPcmG50Mkjt4g7wToDzHna8avE2XKx/0zyyQi+GA
|
29
|
+
/9NFpKTgNbmj/FKnswGGBn25jOUyhEhbhIIapNL63ZLfS3t1SCFc7ocXgTHY2jF4
|
30
|
+
JFE9pOieKsAjSNbLcEMCfrI+TWAJRCarkLmzvo/IsyXrMJZzW4lxQmzskfxV0/ta
|
31
|
+
y1wbhwD9baEJBoJV0HJUYmUKv0o4IOdPWUtz4muYo4mBGOUMXy2BKFh/Cv6Dx2OV
|
32
|
+
VHCW1SMgyzGgpF4w+Jy/+hxNRcJPfB1bhnklR4mctWPR6UuMLn2nhWhttoO29UGt
|
33
|
+
qk9INcfDhzlxwVIqYAoLXTV/ru3mTZI=
|
34
|
+
-----END CERTIFICATE-----
|