logstash-filter-elastic_integration 0.0.1-java → 0.0.2-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/logstash/filters/elastic_integration/jar_dependencies.rb +1 -1
- data/lib/logstash/filters/elastic_integration.rb +126 -57
- data/vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/{0.0.1/logstash-filter-elastic_integration-0.0.1.jar → 0.0.2/logstash-filter-elastic_integration-0.0.2.jar} +0 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e763ac8238ed2ebca641f89dde3596d76b63f2e93f9214894c93add947c0fb14
|
4
|
+
data.tar.gz: acb232f4b3481f47684421ccfffdda198a4141ec5209b67f881f04ca28edd539
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d563ccc980501c370b2b95eecbfbd8f43ca9cc30d792d3ab8122cc08bfd227c1a4bfa41dcea7041f502afb25d4a3f391981309157b5a5bddf8b92a37543167f3
|
7
|
+
data.tar.gz: 3e75a0255e24a4c67ef04f65e7bdeb137c233d9c731624b0436c26a3f1ffe0bf6274b52090a924b072f602678b6af7570c105f590b81e13edf61bfdff1df8936
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.2
|
@@ -8,4 +8,4 @@
|
|
8
8
|
########################################################################
|
9
9
|
|
10
10
|
require 'jar_dependencies'
|
11
|
-
require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.
|
11
|
+
require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.2')
|
@@ -10,21 +10,10 @@
|
|
10
10
|
require "logstash/filters/base"
|
11
11
|
require "logstash/namespace"
|
12
12
|
|
13
|
-
require_relative "elastic_integration/jar_dependencies"
|
14
|
-
|
15
13
|
class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
16
14
|
|
17
|
-
require_relative "elastic_integration/event_api_bridge"
|
18
|
-
include EventApiBridge
|
19
|
-
|
20
15
|
config_name "elastic_integration"
|
21
16
|
|
22
|
-
java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
|
23
|
-
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessor')
|
24
|
-
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
|
25
|
-
java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
|
26
|
-
java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
|
27
|
-
|
28
17
|
ELASTICSEARCH_DEFAULT_PORT = 9200.freeze
|
29
18
|
ELASTICSEARCH_DEFAULT_PATH = '/'.freeze
|
30
19
|
HTTP_PROTOCOL = "http".freeze
|
@@ -86,10 +75,10 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
86
75
|
config :ssl_keystore_password, :validate => :password
|
87
76
|
|
88
77
|
# Username for basic authentication
|
89
|
-
config :
|
78
|
+
config :username, :validate => :string
|
90
79
|
|
91
80
|
# Password for basic authentication
|
92
|
-
config :
|
81
|
+
config :password, :validate => :password
|
93
82
|
|
94
83
|
# Cloud authentication string ("<username>:<password>" format) to connect to Elastic cloud.
|
95
84
|
#
|
@@ -99,23 +88,30 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
99
88
|
# A key to authenticate when connecting to Elasticsearch
|
100
89
|
config :api_key, :validate => :password
|
101
90
|
|
102
|
-
# A directory containing one or more Maxmind
|
91
|
+
# A directory containing one or more Maxmind Database files in *.mmdb format
|
103
92
|
config :geoip_database_directory, :validate => :path
|
104
93
|
|
94
|
+
# a sprintf template for resolving the pipeline name; when this template does
|
95
|
+
# not fully-resolve no pipeline will be run.
|
96
|
+
config :pipeline_name, :validate => :string
|
105
97
|
|
98
|
+
##
|
99
|
+
# Validates that this plugin can be initialized BEFORE loading dependencies
|
100
|
+
# and delegating to super, so that when this plugin CANNOT be run the process
|
101
|
+
# is not encumbered by those dependencies.
|
106
102
|
def initialize(*a, &b)
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
that REQUIRES the complete Logstash distribution, including non-OSS features.
|
115
|
-
ERR
|
116
|
-
end
|
103
|
+
ensure_complete_logstash!
|
104
|
+
ensure_java_major_version!(17)
|
105
|
+
|
106
|
+
require_relative "elastic_integration/jar_dependencies"
|
107
|
+
require_relative "elastic_integration/event_api_bridge"
|
108
|
+
|
109
|
+
extend EventApiBridge
|
117
110
|
|
118
111
|
super
|
112
|
+
|
113
|
+
java_import('co.elastic.logstash.filters.elasticintegration.util.PluginContext')
|
114
|
+
@plugin_context = PluginContext.new(execution_context&.pipeline_id || "UNDEF", id)
|
119
115
|
end
|
120
116
|
|
121
117
|
def register
|
@@ -207,16 +203,16 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
207
203
|
def validate_auth_settings!
|
208
204
|
@cloud_auth = @cloud_auth&.freeze
|
209
205
|
@api_key = @api_key&.freeze
|
210
|
-
@
|
211
|
-
@
|
206
|
+
@username = @username&.freeze
|
207
|
+
@password = @password&.freeze
|
212
208
|
|
213
|
-
raise_config_error! "`
|
214
|
-
raise_config_error! "`
|
215
|
-
if @
|
216
|
-
raise_config_error! "Empty `
|
209
|
+
raise_config_error! "`username` requires `password`" if @username && !@password
|
210
|
+
raise_config_error! "`password` is not allowed unless `username` is specified" if !@username && @password
|
211
|
+
if @username && @password
|
212
|
+
raise_config_error! "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty?
|
217
213
|
end
|
218
214
|
|
219
|
-
possible_auth_options = original_params.keys & %w(
|
215
|
+
possible_auth_options = original_params.keys & %w(password cloud_auth api_key)
|
220
216
|
raise_config_error!("Multiple authentication #{possible_auth_options} options cannot be used together. Please provide ONLY one.") if possible_auth_options.size > 1
|
221
217
|
|
222
218
|
raise_config_error! "Empty `cloud_auth` is not allowed" if @cloud_auth && @cloud_auth.value.empty?
|
@@ -303,38 +299,43 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
303
299
|
##
|
304
300
|
# Builds a `PluginConfiguration` from the previously-validated config
|
305
301
|
def extract_immutable_config
|
306
|
-
|
302
|
+
java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
|
303
|
+
@immutable_config ||= PluginConfiguration::Builder.new.tap do |builder|
|
304
|
+
builder.setId @id
|
307
305
|
|
308
|
-
|
306
|
+
builder.setHosts @hosts&.map(&:to_s)
|
307
|
+
builder.setCloudId @cloud_id
|
309
308
|
|
310
|
-
|
311
|
-
builder.setCloudId @cloud_id
|
309
|
+
builder.setSslEnabled @ssl_enabled
|
312
310
|
|
313
|
-
|
311
|
+
# ssl trust
|
312
|
+
builder.setSslVerificationMode @ssl_verification_mode
|
313
|
+
builder.setSslTruststorePath @ssl_truststore_path
|
314
|
+
builder.setSslTruststorePassword @ssl_truststore_password
|
315
|
+
builder.setSslCertificateAuthorities @ssl_certificate_authorities
|
314
316
|
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
317
|
+
# ssl identity
|
318
|
+
builder.setSslKeystorePath @keystore
|
319
|
+
builder.setSslKeystorePassword @ssl_keystore_password
|
320
|
+
builder.setSslCertificate @ssl_certificate
|
321
|
+
builder.setSslKey @ssl_key
|
322
|
+
builder.setSslKeyPassphrase @ssl_key_passphrase
|
320
323
|
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
builder.setSslKeyPassphrase @ssl_key_passphrase
|
324
|
+
# request auth
|
325
|
+
builder.setAuthBasicUsername @username
|
326
|
+
builder.setAuthBasicPassword @password
|
327
|
+
builder.setCloudAuth @cloud_auth
|
328
|
+
builder.setApiKey @api_key
|
327
329
|
|
328
|
-
|
329
|
-
|
330
|
-
builder.setAuthBasicPassword @auth_basic_password
|
331
|
-
builder.setCloudAuth @cloud_auth
|
332
|
-
builder.setApiKey @api_key
|
330
|
+
# pipeline resolving
|
331
|
+
builder.setPipelineNameTemplate @pipeline_name
|
333
332
|
|
334
|
-
|
333
|
+
end.build
|
335
334
|
end
|
336
335
|
|
337
336
|
def initialize_elasticsearch_rest_client!
|
337
|
+
java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
|
338
|
+
|
338
339
|
@elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(extract_immutable_config)
|
339
340
|
.map(&:build)
|
340
341
|
.orElseThrow() # todo: ruby/java bridge better exception
|
@@ -344,17 +345,16 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
344
345
|
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
|
345
346
|
java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpProcessorFactory')
|
346
347
|
|
347
|
-
@event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client)
|
348
|
+
@event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client, extract_immutable_config)
|
348
349
|
.setFilterMatchListener(method(:filter_matched_java).to_proc)
|
349
350
|
.addProcessor("geoip") { GeoIpProcessorFactory.new(@geoip_database_provider) }
|
350
|
-
.build(
|
351
|
+
.build(@plugin_context)
|
351
352
|
rescue => exception
|
352
353
|
raise_config_error!("configuration did not produce an EventProcessor: #{exception}")
|
353
354
|
end
|
354
355
|
|
355
356
|
def initialize_geoip_database_provider!
|
356
357
|
java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpDatabaseProvider')
|
357
|
-
java_import('co.elastic.logstash.filters.elasticintegration.geoip.StaticGeoIpDatabase')
|
358
358
|
|
359
359
|
@geoip_database_provider ||= GeoIpDatabaseProvider::Builder.new.tap do |builder|
|
360
360
|
builder.setDatabases(java.io.File.new(@geoip_database_directory)) if @geoip_database_directory
|
@@ -362,8 +362,77 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
362
362
|
end
|
363
363
|
|
364
364
|
def perform_preflight_check!
|
365
|
-
PreflightCheck
|
365
|
+
java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
|
366
|
+
|
367
|
+
check_user_privileges!
|
368
|
+
check_es_cluster_license!
|
369
|
+
end
|
370
|
+
|
371
|
+
def check_user_privileges!
|
372
|
+
PreflightCheck.new(@elasticsearch_rest_client).checkUserPrivileges
|
373
|
+
rescue => e
|
374
|
+
security_error_message = "no handler found for uri [/_security/user/_has_privileges]"
|
375
|
+
if e.message.include?(security_error_message)
|
376
|
+
if @password || @cloud_auth || @api_key
|
377
|
+
cred_desc = case
|
378
|
+
when @password then "`username` and `password`"
|
379
|
+
when @cloud_auth then "`cloud_auth`"
|
380
|
+
when @api_key then "`api_key`"
|
381
|
+
end
|
382
|
+
|
383
|
+
recommend_message = "The Elasticsearch cluster does not have security features enabled but request credentials were provided. Either enable security in Elasticsearch (recommended!) or remove the #{cred_desc} request credentials. "
|
384
|
+
raise_config_error! recommend_message.concat(e.message)
|
385
|
+
else
|
386
|
+
# Elasticsearch cluster security disabled, auth also isn't provided, running plugin unsecurily
|
387
|
+
@logger.warn("`elastic_integration` plugin is unable to verify user privileges. It has started with unsafe mode which may cause unexpected failure. Enabling security in Elasticsearch and using user authentication is recommended.")
|
388
|
+
end
|
389
|
+
else
|
390
|
+
raise_config_error!(e.message)
|
391
|
+
end
|
392
|
+
end
|
393
|
+
|
394
|
+
def check_es_cluster_license!
|
395
|
+
PreflightCheck.new(@elasticsearch_rest_client).checkLicense
|
366
396
|
rescue => e
|
367
397
|
raise_config_error!(e.message)
|
368
398
|
end
|
399
|
+
|
400
|
+
##
|
401
|
+
# single-use helper to ensure the running Logstash is a _complete_ distro that has
|
402
|
+
# non-OSS features active. Runtime detection mechanism relies on LogStash::OSS,
|
403
|
+
# which is set in the prelude to LogStash::Runner, and is bypassed when LogStash::OSS
|
404
|
+
# is not defined (such as when running specs from source)
|
405
|
+
def ensure_complete_logstash!
|
406
|
+
if defined?(LogStash::OSS) && LogStash::OSS
|
407
|
+
raise_config_error! <<~ERR.gsub(/\s+/, ' ')
|
408
|
+
The Elastic Integration filter for Logstash is an Elastic-licensed plugin
|
409
|
+
that REQUIRES the complete Logstash distribution, including non-OSS features.
|
410
|
+
ERR
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
##
|
415
|
+
# single-use helper for ensuring the running JVM meets the minimum
|
416
|
+
# Java version requirement necessary for loading the included jars
|
417
|
+
# @raise [LogStash::EnvironmentError]
|
418
|
+
def ensure_java_major_version!(minimum_major_version)
|
419
|
+
java_version_string = java.lang.System.getProperty("java.specification.version")
|
420
|
+
|
421
|
+
# MAJOR <= 8 ? 1.MAJOR.MINOR : MAJOR.MINOR
|
422
|
+
# https://rubular.com/r/lLW5iUWN9N9N6Z
|
423
|
+
java_major_version_pattern = /(?:(?:(?<=^1\.)[0-8])|^(?:9|[1-9][0-9]+))(?=\.|$)/
|
424
|
+
java_major_version = java_version_string&.slice(java_major_version_pattern)&.to_i
|
425
|
+
|
426
|
+
if (java_major_version.nil?)
|
427
|
+
fail("Failed to retrieve running JVM's major version")
|
428
|
+
elsif (java_major_version < minimum_major_version)
|
429
|
+
fail(LogStash::EnvironmentError, <<~ERR.gsub(/\s+/, ' '))
|
430
|
+
the #{self.class.config_name} #{self.class.plugin_type} plugin requires
|
431
|
+
Java #{minimum_major_version} or later and cannot be instantiated on the
|
432
|
+
current JVM version `#{java_version_string}`.
|
433
|
+
You can either remove the plugin from your pipeline definition or run
|
434
|
+
Logstash with a supported JVM.
|
435
|
+
ERR
|
436
|
+
end
|
437
|
+
end
|
369
438
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elastic_integration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,7 +73,7 @@ files:
|
|
73
73
|
- lib/logstash/filters/elastic_integration/event_api_bridge.rb
|
74
74
|
- lib/logstash/filters/elastic_integration/jar_dependencies.rb
|
75
75
|
- logstash-filter-elastic_integration.gemspec
|
76
|
-
- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.
|
76
|
+
- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.2/logstash-filter-elastic_integration-0.0.2.jar
|
77
77
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
78
78
|
licenses:
|
79
79
|
- ELv2
|
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
100
|
- !ruby/object:Gem::Version
|
101
101
|
version: '0'
|
102
102
|
requirements: []
|
103
|
-
rubygems_version: 3.2.
|
103
|
+
rubygems_version: 3.2.29
|
104
104
|
signing_key:
|
105
105
|
specification_version: 4
|
106
106
|
summary: Processes Elastic Integrations
|