logstash-filter-elastic_integration 0.0.1-java → 0.0.3-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/logstash/filters/elastic_integration/jar_dependencies.rb +1 -1
- data/lib/logstash/filters/elastic_integration.rb +145 -58
- data/vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/{0.0.1/logstash-filter-elastic_integration-0.0.1.jar → 0.0.3/logstash-filter-elastic_integration-0.0.3.jar} +0 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 245850625d596870d7536a1282f78419735370cf2d4b4a4d5ad51d59872f7de9
|
4
|
+
data.tar.gz: '017508efe2333faa598921431926a80f58cfaf7f2dae9c6b5a2d53a787bc296b'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3234009681fad3833eacd39a943661a61ae4541c4d4fe3df6f5c0f0516ca53bdf6da70eaf5890de460def6c231173c785c1a9a317c82be0275dc8ee79a99cf81
|
7
|
+
data.tar.gz: f5bc0d1effe8c21c5c7cdc69ffe265d8cbcc10f67ee32e4e64d43ea4d494618db78edf8d5e42d05e644bfb87d2862982fddc8df571f0d366a6bcb6685b16e545
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.3
|
@@ -8,4 +8,4 @@
|
|
8
8
|
########################################################################
|
9
9
|
|
10
10
|
require 'jar_dependencies'
|
11
|
-
require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.
|
11
|
+
require_jar('co.elastic.logstash.plugins.filter.elasticintegration', 'logstash-filter-elastic_integration', '0.0.3')
|
@@ -10,25 +10,15 @@
|
|
10
10
|
require "logstash/filters/base"
|
11
11
|
require "logstash/namespace"
|
12
12
|
|
13
|
-
require_relative "elastic_integration/jar_dependencies"
|
14
|
-
|
15
13
|
class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
16
14
|
|
17
|
-
require_relative "elastic_integration/event_api_bridge"
|
18
|
-
include EventApiBridge
|
19
|
-
|
20
15
|
config_name "elastic_integration"
|
21
16
|
|
22
|
-
java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
|
23
|
-
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessor')
|
24
|
-
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
|
25
|
-
java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
|
26
|
-
java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
|
27
|
-
|
28
17
|
ELASTICSEARCH_DEFAULT_PORT = 9200.freeze
|
29
18
|
ELASTICSEARCH_DEFAULT_PATH = '/'.freeze
|
30
19
|
HTTP_PROTOCOL = "http".freeze
|
31
20
|
HTTPS_PROTOCOL = "https".freeze
|
21
|
+
ELASTIC_API_VERSION = "2023-10-31".freeze
|
32
22
|
|
33
23
|
# Sets the host(s) of the remote instance. If given an array it will load balance
|
34
24
|
# requests across the hosts specified in the `hosts` parameter. Hosts can be any of
|
@@ -86,10 +76,10 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
86
76
|
config :ssl_keystore_password, :validate => :password
|
87
77
|
|
88
78
|
# Username for basic authentication
|
89
|
-
config :
|
79
|
+
config :username, :validate => :string
|
90
80
|
|
91
81
|
# Password for basic authentication
|
92
|
-
config :
|
82
|
+
config :password, :validate => :password
|
93
83
|
|
94
84
|
# Cloud authentication string ("<username>:<password>" format) to connect to Elastic cloud.
|
95
85
|
#
|
@@ -99,23 +89,30 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
99
89
|
# A key to authenticate when connecting to Elasticsearch
|
100
90
|
config :api_key, :validate => :password
|
101
91
|
|
102
|
-
# A directory containing one or more Maxmind
|
92
|
+
# A directory containing one or more Maxmind Database files in *.mmdb format
|
103
93
|
config :geoip_database_directory, :validate => :path
|
104
94
|
|
95
|
+
# a sprintf template for resolving the pipeline name; when this template does
|
96
|
+
# not fully-resolve no pipeline will be run.
|
97
|
+
config :pipeline_name, :validate => :string
|
105
98
|
|
99
|
+
##
|
100
|
+
# Validates that this plugin can be initialized BEFORE loading dependencies
|
101
|
+
# and delegating to super, so that when this plugin CANNOT be run the process
|
102
|
+
# is not encumbered by those dependencies.
|
106
103
|
def initialize(*a, &b)
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
that REQUIRES the complete Logstash distribution, including non-OSS features.
|
115
|
-
ERR
|
116
|
-
end
|
104
|
+
ensure_complete_logstash!
|
105
|
+
ensure_java_major_version!(17)
|
106
|
+
|
107
|
+
require_relative "elastic_integration/jar_dependencies"
|
108
|
+
require_relative "elastic_integration/event_api_bridge"
|
109
|
+
|
110
|
+
extend EventApiBridge
|
117
111
|
|
118
112
|
super
|
113
|
+
|
114
|
+
java_import('co.elastic.logstash.filters.elasticintegration.util.PluginContext')
|
115
|
+
@plugin_context = PluginContext.new(execution_context&.pipeline_id || "UNDEF", id)
|
119
116
|
end
|
120
117
|
|
121
118
|
def register
|
@@ -207,16 +204,16 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
207
204
|
def validate_auth_settings!
|
208
205
|
@cloud_auth = @cloud_auth&.freeze
|
209
206
|
@api_key = @api_key&.freeze
|
210
|
-
@
|
211
|
-
@
|
207
|
+
@username = @username&.freeze
|
208
|
+
@password = @password&.freeze
|
212
209
|
|
213
|
-
raise_config_error! "`
|
214
|
-
raise_config_error! "`
|
215
|
-
if @
|
216
|
-
raise_config_error! "Empty `
|
210
|
+
raise_config_error! "`username` requires `password`" if @username && !@password
|
211
|
+
raise_config_error! "`password` is not allowed unless `username` is specified" if !@username && @password
|
212
|
+
if @username && @password
|
213
|
+
raise_config_error! "Empty `username` or `password` is not allowed" if @username.empty? || @password.value.empty?
|
217
214
|
end
|
218
215
|
|
219
|
-
possible_auth_options = original_params.keys & %w(
|
216
|
+
possible_auth_options = original_params.keys & %w(password cloud_auth api_key)
|
220
217
|
raise_config_error!("Multiple authentication #{possible_auth_options} options cannot be used together. Please provide ONLY one.") if possible_auth_options.size > 1
|
221
218
|
|
222
219
|
raise_config_error! "Empty `cloud_auth` is not allowed" if @cloud_auth && @cloud_auth.value.empty?
|
@@ -303,58 +300,73 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
303
300
|
##
|
304
301
|
# Builds a `PluginConfiguration` from the previously-validated config
|
305
302
|
def extract_immutable_config
|
306
|
-
|
303
|
+
java_import('co.elastic.logstash.filters.elasticintegration.PluginConfiguration')
|
304
|
+
@immutable_config ||= PluginConfiguration::Builder.new.tap do |builder|
|
305
|
+
builder.setId @id
|
307
306
|
|
308
|
-
|
307
|
+
builder.setHosts @hosts&.map(&:to_s)
|
308
|
+
builder.setCloudId @cloud_id
|
309
309
|
|
310
|
-
|
311
|
-
builder.setCloudId @cloud_id
|
310
|
+
builder.setSslEnabled @ssl_enabled
|
312
311
|
|
313
|
-
|
312
|
+
# ssl trust
|
313
|
+
builder.setSslVerificationMode @ssl_verification_mode
|
314
|
+
builder.setSslTruststorePath @ssl_truststore_path
|
315
|
+
builder.setSslTruststorePassword @ssl_truststore_password
|
316
|
+
builder.setSslCertificateAuthorities @ssl_certificate_authorities
|
314
317
|
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
318
|
+
# ssl identity
|
319
|
+
builder.setSslKeystorePath @keystore
|
320
|
+
builder.setSslKeystorePassword @ssl_keystore_password
|
321
|
+
builder.setSslCertificate @ssl_certificate
|
322
|
+
builder.setSslKey @ssl_key
|
323
|
+
builder.setSslKeyPassphrase @ssl_key_passphrase
|
320
324
|
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
builder.setSslKeyPassphrase @ssl_key_passphrase
|
325
|
+
# request auth
|
326
|
+
builder.setAuthBasicUsername @username
|
327
|
+
builder.setAuthBasicPassword @password
|
328
|
+
builder.setCloudAuth @cloud_auth
|
329
|
+
builder.setApiKey @api_key
|
327
330
|
|
328
|
-
|
329
|
-
|
330
|
-
builder.setAuthBasicPassword @auth_basic_password
|
331
|
-
builder.setCloudAuth @cloud_auth
|
332
|
-
builder.setApiKey @api_key
|
331
|
+
# pipeline resolving
|
332
|
+
builder.setPipelineNameTemplate @pipeline_name
|
333
333
|
|
334
|
-
|
334
|
+
end.build
|
335
335
|
end
|
336
336
|
|
337
337
|
def initialize_elasticsearch_rest_client!
|
338
|
-
|
338
|
+
java_import('co.elastic.logstash.filters.elasticintegration.ElasticsearchRestClientBuilder')
|
339
|
+
java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
|
340
|
+
|
341
|
+
config = extract_immutable_config
|
342
|
+
@elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(config)
|
339
343
|
.map(&:build)
|
340
344
|
.orElseThrow() # todo: ruby/java bridge better exception
|
345
|
+
|
346
|
+
if serverless?
|
347
|
+
@elasticsearch_rest_client = ElasticsearchRestClientBuilder.fromPluginConfiguration(config)
|
348
|
+
.map do |builder|
|
349
|
+
builder.configureElasticApi { |elasticApi| elasticApi.setApiVersion(ELASTIC_API_VERSION) }
|
350
|
+
end
|
351
|
+
.map(&:build)
|
352
|
+
.orElseThrow()
|
353
|
+
end
|
341
354
|
end
|
342
355
|
|
343
356
|
def initialize_event_processor!
|
344
357
|
java_import('co.elastic.logstash.filters.elasticintegration.EventProcessorBuilder')
|
345
358
|
java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpProcessorFactory')
|
346
359
|
|
347
|
-
@event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client)
|
360
|
+
@event_processor = EventProcessorBuilder.fromElasticsearch(@elasticsearch_rest_client, extract_immutable_config)
|
348
361
|
.setFilterMatchListener(method(:filter_matched_java).to_proc)
|
349
362
|
.addProcessor("geoip") { GeoIpProcessorFactory.new(@geoip_database_provider) }
|
350
|
-
.build(
|
363
|
+
.build(@plugin_context)
|
351
364
|
rescue => exception
|
352
365
|
raise_config_error!("configuration did not produce an EventProcessor: #{exception}")
|
353
366
|
end
|
354
367
|
|
355
368
|
def initialize_geoip_database_provider!
|
356
369
|
java_import('co.elastic.logstash.filters.elasticintegration.geoip.GeoIpDatabaseProvider')
|
357
|
-
java_import('co.elastic.logstash.filters.elasticintegration.geoip.StaticGeoIpDatabase')
|
358
370
|
|
359
371
|
@geoip_database_provider ||= GeoIpDatabaseProvider::Builder.new.tap do |builder|
|
360
372
|
builder.setDatabases(java.io.File.new(@geoip_database_directory)) if @geoip_database_directory
|
@@ -362,8 +374,83 @@ class LogStash::Filters::ElasticIntegration < LogStash::Filters::Base
|
|
362
374
|
end
|
363
375
|
|
364
376
|
def perform_preflight_check!
|
365
|
-
PreflightCheck
|
377
|
+
java_import('co.elastic.logstash.filters.elasticintegration.PreflightCheck')
|
378
|
+
|
379
|
+
check_user_privileges!
|
380
|
+
check_es_cluster_license!
|
381
|
+
end
|
382
|
+
|
383
|
+
def check_user_privileges!
|
384
|
+
PreflightCheck.new(@elasticsearch_rest_client).checkUserPrivileges
|
385
|
+
rescue => e
|
386
|
+
security_error_message = "no handler found for uri [/_security/user/_has_privileges]"
|
387
|
+
if e.message.include?(security_error_message)
|
388
|
+
if @password || @cloud_auth || @api_key
|
389
|
+
cred_desc = case
|
390
|
+
when @password then "`username` and `password`"
|
391
|
+
when @cloud_auth then "`cloud_auth`"
|
392
|
+
when @api_key then "`api_key`"
|
393
|
+
end
|
394
|
+
|
395
|
+
recommend_message = "The Elasticsearch cluster does not have security features enabled but request credentials were provided. Either enable security in Elasticsearch (recommended!) or remove the #{cred_desc} request credentials. "
|
396
|
+
raise_config_error! recommend_message.concat(e.message)
|
397
|
+
else
|
398
|
+
# Elasticsearch cluster security disabled, auth also isn't provided, running plugin unsecurily
|
399
|
+
@logger.warn("`elastic_integration` plugin is unable to verify user privileges. It has started with unsafe mode which may cause unexpected failure. Enabling security in Elasticsearch and using user authentication is recommended.")
|
400
|
+
end
|
401
|
+
else
|
402
|
+
raise_config_error!(e.message)
|
403
|
+
end
|
404
|
+
end
|
405
|
+
|
406
|
+
def check_es_cluster_license!
|
407
|
+
PreflightCheck.new(@elasticsearch_rest_client).checkLicense
|
366
408
|
rescue => e
|
367
409
|
raise_config_error!(e.message)
|
368
410
|
end
|
411
|
+
|
412
|
+
def serverless?
|
413
|
+
PreflightCheck.new(@elasticsearch_rest_client).isServerless
|
414
|
+
rescue => e
|
415
|
+
raise_config_error!(e.message)
|
416
|
+
end
|
417
|
+
|
418
|
+
##
|
419
|
+
# single-use helper to ensure the running Logstash is a _complete_ distro that has
|
420
|
+
# non-OSS features active. Runtime detection mechanism relies on LogStash::OSS,
|
421
|
+
# which is set in the prelude to LogStash::Runner, and is bypassed when LogStash::OSS
|
422
|
+
# is not defined (such as when running specs from source)
|
423
|
+
def ensure_complete_logstash!
|
424
|
+
if defined?(LogStash::OSS) && LogStash::OSS
|
425
|
+
raise_config_error! <<~ERR.gsub(/\s+/, ' ')
|
426
|
+
The Elastic Integration filter for Logstash is an Elastic-licensed plugin
|
427
|
+
that REQUIRES the complete Logstash distribution, including non-OSS features.
|
428
|
+
ERR
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
432
|
+
##
|
433
|
+
# single-use helper for ensuring the running JVM meets the minimum
|
434
|
+
# Java version requirement necessary for loading the included jars
|
435
|
+
# @raise [LogStash::EnvironmentError]
|
436
|
+
def ensure_java_major_version!(minimum_major_version)
|
437
|
+
java_version_string = java.lang.System.getProperty("java.specification.version")
|
438
|
+
|
439
|
+
# MAJOR <= 8 ? 1.MAJOR.MINOR : MAJOR.MINOR
|
440
|
+
# https://rubular.com/r/lLW5iUWN9N9N6Z
|
441
|
+
java_major_version_pattern = /(?:(?:(?<=^1\.)[0-8])|^(?:9|[1-9][0-9]+))(?=\.|$)/
|
442
|
+
java_major_version = java_version_string&.slice(java_major_version_pattern)&.to_i
|
443
|
+
|
444
|
+
if (java_major_version.nil?)
|
445
|
+
fail("Failed to retrieve running JVM's major version")
|
446
|
+
elsif (java_major_version < minimum_major_version)
|
447
|
+
fail(LogStash::EnvironmentError, <<~ERR.gsub(/\s+/, ' '))
|
448
|
+
the #{self.class.config_name} #{self.class.plugin_type} plugin requires
|
449
|
+
Java #{minimum_major_version} or later and cannot be instantiated on the
|
450
|
+
current JVM version `#{java_version_string}`.
|
451
|
+
You can either remove the plugin from your pipeline definition or run
|
452
|
+
Logstash with a supported JVM.
|
453
|
+
ERR
|
454
|
+
end
|
455
|
+
end
|
369
456
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elastic_integration
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-04
|
11
|
+
date: 2023-10-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -73,7 +73,7 @@ files:
|
|
73
73
|
- lib/logstash/filters/elastic_integration/event_api_bridge.rb
|
74
74
|
- lib/logstash/filters/elastic_integration/jar_dependencies.rb
|
75
75
|
- logstash-filter-elastic_integration.gemspec
|
76
|
-
- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.
|
76
|
+
- vendor/jar-dependencies/co/elastic/logstash/plugins/filter/elasticintegration/logstash-filter-elastic_integration/0.0.3/logstash-filter-elastic_integration-0.0.3.jar
|
77
77
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
78
78
|
licenses:
|
79
79
|
- ELv2
|