logstash-filter-elastic_integration 0.0.1-java → 0.0.2-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/logstash/filters/elastic_integration/jar_dependencies.rb +1 -1
- data/lib/logstash/filters/elastic_integration.rb +126 -57
- data/vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/{0.0.1/logstash-filter-elastic_integration-0.0.1.jar → 0.0.2/logstash-filter-elastic_integration-0.0.2.jar} +0 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e763ac8238ed2ebca641f89dde3596d76b63f2e93f9214894c93add947c0fb14
|
4
|
+
data.tar.gz: acb232f4b3481f47684421ccfffdda198a4141ec5209b67f881f04ca28edd539
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d563ccc980501c370b2b95eecbfbd8f43ca9cc30d792d3ab8122cc08bfd227c1a4bfa41dcea7041f502afb25d4a3f391981309157b5a5bddf8b92a37543167f3
|
7
|
+
data.tar.gz: 3e75a0255e24a4c67ef04f65e7bdeb137c233d9c731624b0436c26a3f1ffe0bf6274b52090a924b072f602678b6af7570c105f590b81e13edf61bfdff1df8936
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.2
|
@@ -8,4 +8,4 @@
|
|
8
8
|
########################################################################
|
9
9
|
|
10
10
|
require 'jar_dependencies'
|
11
|
-
require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.
|
11
|
+
require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.2')
|
@@ -10,21 +10,10 @@
|
|
10
10
|
require "logstash/filters/base"
|
11
11
|
require "logstash/namespace"
|
12
12
|
|
13
|
-
require_relative "elastic_integration/jar_dependencies"
|
14
|
-
|
15
13
|
class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
16
14
|
|
17
|
-
require_relative "elastic_integration/event_api_bridge"
|
18
|
-
include EventApiBridge
|
19
|
-
|
20
15
|
config_name "elastic_integration"
|
21
16
|
|
22
|
-
java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
|
23
|
-
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessor')
|
24
|
-
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
|
25
|
-
java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
|
26
|
-
java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
|
27
|
-
|
28
17
|
ELASTICSEARCH_DEFAULT_PORT = 9200.freeze
|
29
18
|
ELASTICSEARCH_DEFAULT_PATH = '/'.freeze
|
30
19
|
HTTP_PROTOCOL = "http".freeze
|
@@ -86,10 +75,10 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
86
75
|
config :ssl_keystore_password, :validate => :password
|
87
76
|
|
88
77
|
# Username for basic authentication
|
89
|
-
config :
|
78
|
+
config :username, :validate => :string
|
90
79
|
|
91
80
|
# Password for basic authentication
|
92
|
-
config :
|
81
|
+
config :password, :validate => :password
|
93
82
|
|
94
83
|
# Cloud authentication string ("<username>:<password>" format) to connect to Elastic cloud.
|
95
84
|
#
|
@@ -99,23 +88,30 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
99
88
|
# A key to authenticate when connecting to Elasticsearch
|
100
89
|
config :api_key, :validate => :password
|
101
90
|
|
102
|
-
# A directory containing one or more Maxmind
|
91
|
+
# A directory containing one or more Maxmind Database files in *.mmdb format
|
103
92
|
config :geoip_database_directory, :validate => :path
|
104
93
|
|
94
|
+
# a sprintf template for resolving the pipeline name; when this template does
|
95
|
+
# not fully-resolve no pipeline will be run.
|
96
|
+
config :pipeline_name, :validate => :string
|
105
97
|
|
98
|
+
##
|
99
|
+
# Validates that this plugin can be initialized BEFORE loading dependencies
|
100
|
+
# and delegating to super, so that when this plugin CANNOT be run the process
|
101
|
+
# is not encumbered by those dependencies.
|
106
102
|
def initialize(*a, &b)
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
that REQUIRES the complete Logstash distribution, including non-OSS features.
|
115
|
-
ERR
|
116
|
-
end
|
103
|
+
ensure_complete_logstash!
|
104
|
+
ensure_java_major_version!(17)
|
105
|
+
|
106
|
+
require_relative "elastic_integration/jar_dependencies"
|
107
|
+
require_relative "elastic_integration/event_api_bridge"
|
108
|
+
|
109
|
+
extend EventApiBridge
|
117
110
|
|
118
111
|
super
|
112
|
+
|
113
|
+
java_import('co.elastic.logstash.filters.elasticintegration.util.PluginContext')
|
114
|
+
@plugin_context = PluginContext.new(execution_context&.pipeline_id || "UNDEF", id)
|
119
115
|
end
|
120
116
|
|
121
117
|
def register
|
@@ -207,16 +203,16 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
207
203
|
def validate_auth_settings!
|
208
204
|
@cloud_auth = @cloud_auth&.freeze
|
209
205
|
@api_key = @api_key&.freeze
|
210
|
-
@
|
211
|
-
@
|
206
|
+
@username = @username&.freeze
|
207
|
+
@password = @password&.freeze
|
212
208
|
|
213
|
-
raise_config_error! "`
|
214
|
-
raise_config_error! "`
|
215
|
-
if @
|
216
|
-
raise_config_error! "Empty `
|
209
|
+
raise_config_error! "`username` requires `password`" if @username && !@password
|
210
|
+
raise_config_error! "`password` is not allowed unless `username` is specified" if !@username && @password
|
211
|
+
if @username && @password
|
212
|
+
raise_config_error! "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty?
|
217
213
|
end
|
218
214
|
|
219
|
-
possible_auth_options = original_params.keys & %w(
|
215
|
+
possible_auth_options = original_params.keys & %w(password cloud_auth api_key)
|
220
216
|
raise_config_error!("Multiple authentication #{possible_auth_options} options cannot be used together. Please provide ONLY one.") if possible_auth_options.size > 1
|
221
217
|
|
222
218
|
raise_config_error! "Empty `cloud_auth` is not allowed" if @cloud_auth && @cloud_auth.value.empty?
|
@@ -303,38 +299,43 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
303
299
|
##
|
304
300
|
# Builds a `PluginConfiguration` from the previously-validated config
|
305
301
|
def extract_immutable_config
|
306
|
-
|
302
|
+
java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
|
303
|
+
@immutable_config ||= PluginConfiguration::Builder.new.tap do |builder|
|
304
|
+
builder.setId @id
|
307
305
|
|
308
|
-
|
306
|
+
builder.setHosts @hosts&.map(&:to_s)
|
307
|
+
builder.setCloudId @cloud_id
|
309
308
|
|
310
|
-
|
311
|
-
builder.setCloudId @cloud_id
|
309
|
+
builder.setSslEnabled @ssl_enabled
|
312
310
|
|
313
|
-
|
311
|
+
# ssl trust
|
312
|
+
builder.setSslVerificationMode @ssl_verification_mode
|
313
|
+
builder.setSslTruststorePath @ssl_truststore_path
|
314
|
+
builder.setSslTruststorePassword @ssl_truststore_password
|
315
|
+
builder.setSslCertificateAuthorities @ssl_certificate_authorities
|
314
316
|
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
317
|
+
# ssl identity
|
318
|
+
builder.setSslKeystorePath @keystore
|
319
|
+
builder.setSslKeystorePassword @ssl_keystore_password
|
320
|
+
builder.setSslCertificate @ssl_certificate
|
321
|
+
builder.setSslKey @ssl_key
|
322
|
+
builder.setSslKeyPassphrase @ssl_key_passphrase
|
320
323
|
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
builder.setSslKeyPassphrase @ssl_key_passphrase
|
324
|
+
# request auth
|
325
|
+
builder.setAuthBasicUsername @username
|
326
|
+
builder.setAuthBasicPassword @password
|
327
|
+
builder.setCloudAuth @cloud_auth
|
328
|
+
builder.setApiKey @api_key
|
327
329
|
|
328
|
-
|
329
|
-
|
330
|
-
builder.setAuthBasicPassword @auth_basic_password
|
331
|
-
builder.setCloudAuth @cloud_auth
|
332
|
-
builder.setApiKey @api_key
|
330
|
+
# pipeline resolving
|
331
|
+
builder.setPipelineNameTemplate @pipeline_name
|
333
332
|
|
334
|
-
|
333
|
+
end.build
|
335
334
|
end
|
336
335
|
|
337
336
|
def initialize_elasticsearch_rest_client!
|
337
|
+
java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
|
338
|
+
|
338
339
|
@elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(extract_immutable_config)
|
339
340
|
.map(&:build)
|
340
341
|
.orElseThrow() # todo: ruby/java bridge better exception
|
@@ -344,17 +345,16 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
344
345
|
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
|
345
346
|
java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpProcessorFactory')
|
346
347
|
|
347
|
-
@event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client)
|
348
|
+
@event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client, extract_immutable_config)
|
348
349
|
.setFilterMatchListener(method(:filter_matched_java).to_proc)
|
349
350
|
.addProcessor("geoip") { GeoIpProcessorFactory.new(@geoip_database_provider) }
|
350
|
-
.build(
|
351
|
+
.build(@plugin_context)
|
351
352
|
rescue => exception
|
352
353
|
raise_config_error!("configuration did not produce an EventProcessor: #{exception}")
|
353
354
|
end
|
354
355
|
|
355
356
|
def initialize_geoip_database_provider!
|
356
357
|
java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpDatabaseProvider')
|
357
|
-
java_import('co.elastic.logstash.filters.elasticintegration.geoip.StaticGeoIpDatabase')
|
358
358
|
|
359
359
|
@geoip_database_provider ||= GeoIpDatabaseProvider::Builder.new.tap do |builder|
|
360
360
|
builder.setDatabases(java.io.File.new(@geoip_database_directory)) if @geoip_database_directory
|
@@ -362,8 +362,77 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
362
362
|
end
|
363
363
|
|
364
364
|
def perform_preflight_check!
|
365
|
-
PreflightCheck
|
365
|
+
java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
|
366
|
+
|
367
|
+
check_user_privileges!
|
368
|
+
check_es_cluster_license!
|
369
|
+
end
|
370
|
+
|
371
|
+
def check_user_privileges!
|
372
|
+
PreflightCheck.new(@elasticsearch_rest_client).checkUserPrivileges
|
373
|
+
rescue => e
|
374
|
+
security_error_message = "no handler found for uri [/_security/user/_has_privileges]"
|
375
|
+
if e.message.include?(security_error_message)
|
376
|
+
if @password || @cloud_auth || @api_key
|
377
|
+
cred_desc = case
|
378
|
+
when @password then "`username` and `password`"
|
379
|
+
when @cloud_auth then "`cloud_auth`"
|
380
|
+
when @api_key then "`api_key`"
|
381
|
+
end
|
382
|
+
|
383
|
+
recommend_message = "The Elasticsearch cluster does not have security features enabled but request credentials were provided. Either enable security in Elasticsearch (recommended!) or remove the #{cred_desc} request credentials. "
|
384
|
+
raise_config_error! recommend_message.concat(e.message)
|
385
|
+
else
|
386
|
+
# Elasticsearch cluster security disabled, auth also isn't provided, running plugin unsecurily
|
387
|
+
@logger.warn("`elastic_integration` plugin is unable to verify user privileges. It has started with unsafe mode which may cause unexpected failure. Enabling security in Elasticsearch and using user authentication is recommended.")
|
388
|
+
end
|
389
|
+
else
|
390
|
+
raise_config_error!(e.message)
|
391
|
+
end
|
392
|
+
end
|
393
|
+
|
394
|
+
def check_es_cluster_license!
|
395
|
+
PreflightCheck.new(@elasticsearch_rest_client).checkLicense
|
366
396
|
rescue => e
|
367
397
|
raise_config_error!(e.message)
|
368
398
|
end
|
399
|
+
|
400
|
+
##
|
401
|
+
# single-use helper to ensure the running Logstash is a _complete_ distro that has
|
402
|
+
# non-OSS features active. Runtime detection mechanism relies on LogStash::OSS,
|
403
|
+
# which is set in the prelude to LogStash::Runner, and is bypassed when LogStash::OSS
|
404
|
+
# is not defined (such as when running specs from source)
|
405
|
+
def ensure_complete_logstash!
|
406
|
+
if defined?(LogStash::OSS) && LogStash::OSS
|
407
|
+
raise_config_error! <<~ERR.gsub(/\s+/, ' ')
|
408
|
+
The Elastic Integration filter for Logstash is an Elastic-licensed plugin
|
409
|
+
that REQUIRES the complete Logstash distribution, including non-OSS features.
|
410
|
+
ERR
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
##
|
415
|
+
# single-use helper for ensuring the running JVM meets the minimum
|
416
|
+
# Java version requirement necessary for loading the included jars
|
417
|
+
# @raise [LogStash::EnvironmentError]
|
418
|
+
def ensure_java_major_version!(minimum_major_version)
|
419
|
+
java_version_string = java.lang.System.getProperty("java.specification.version")
|
420
|
+
|
421
|
+
# MAJOR <= 8 ? 1.MAJOR.MINOR : MAJOR.MINOR
|
422
|
+
# https://rubular.com/r/lLW5iUWN9N9N6Z
|
423
|
+
java_major_version_pattern = /(?:(?:(?<=^1\.)[0-8])|^(?:9|[1-9][0-9]+))(?=\.|$)/
|
424
|
+
java_major_version = java_version_string&.slice(java_major_version_pattern)&.to_i
|
425
|
+
|
426
|
+
if (java_major_version.nil?)
|
427
|
+
fail("Failed to retrieve running JVM's major version")
|
428
|
+
elsif (java_major_version < minimum_major_version)
|
429
|
+
fail(LogStash::EnvironmentError, <<~ERR.gsub(/\s+/, ' '))
|
430
|
+
the #{self.class.config_name} #{self.class.plugin_type} plugin requires
|
431
|
+
Java #{minimum_major_version} or later and cannot be instantiated on the
|
432
|
+
current JVM version `#{java_version_string}`.
|
433
|
+
You can either remove the plugin from your pipeline definition or run
|
434
|
+
Logstash with a supported JVM.
|
435
|
+
ERR
|
436
|
+
end
|
437
|
+
end
|
369
438
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elastic_integration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,7 +73,7 @@ files:
|
|
73
73
|
- lib/logstash/filters/elastic_integration/event_api_bridge.rb
|
74
74
|
- lib/logstash/filters/elastic_integration/jar_dependencies.rb
|
75
75
|
- logstash-filter-elastic_integration.gemspec
|
76
|
-
- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.
|
76
|
+
- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.2/logstash-filter-elastic_integration-0.0.2.jar
|
77
77
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
78
78
|
licenses:
|
79
79
|
- ELv2
|
@@ -100,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
100
|
- !ruby/object:Gem::Version
|
101
101
|
version: '0'
|
102
102
|
requirements: []
|
103
|
-
rubygems_version: 3.2.
|
103
|
+
rubygems_version: 3.2.29
|
104
104
|
signing_key:
|
105
105
|
specification_version: 4
|
106
106
|
summary: Processes Elastic Integrations
|