logstash-output-elasticsearch 11.20.1-java → 11.21.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/elasticsearch.rb +26 -6
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/unit/outputs/elasticsearch_spec.rb +152 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 670f82b7bbfea1744a5daaf77a069c8872f87e1b15226131a786f826a931cd9c
|
4
|
+
data.tar.gz: 9aa78b93f19d995562192466ad7b816870706383da69b5050613079a76cd1e92
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 71d5c1b1558c4ba2c0bb29ab761980dfaad7005410a5bda436ac7f1d6a637b523aa058176e14eed491e8ffb99503f963c2eb5a9c3423e3d7ebd550f74902913e
|
7
|
+
data.tar.gz: aa32f8838b2526ae3fbcd3288ef6f848c90d926b0206416c5f2c348855e98494e36721ad9992cfbc238290a7c3e5baa90a518208c4b5cdc6187a80918f7b8716
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 11.21.0
|
2
|
+
- Added support for propagating event processing metadata when this output is downstream of an Elastic Integration Filter and configured _without_ explicit `index`, `document_id`, or `pipeline` directives [#1155](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1155)
|
3
|
+
|
1
4
|
## 11.20.1
|
2
5
|
- Doc: Replace `document_already_exist_exception` with `version_conflict_engine_exception` in the `silence_errors_in_log` setting example [#1159](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1159)
|
3
6
|
|
@@ -540,15 +540,16 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
540
540
|
# @return Hash (initial) parameters for given event
|
541
541
|
# @private shared event params factory between index and data_stream mode
|
542
542
|
def common_event_params(event)
|
543
|
-
|
544
|
-
|
543
|
+
event_control = event.get("[@metadata][_ingest_document]")
|
544
|
+
event_id, event_pipeline, event_index = event_control&.values_at("id","pipeline","index") rescue nil
|
545
|
+
|
545
546
|
params = {
|
546
|
-
:_id =>
|
547
|
-
:_index =>
|
547
|
+
:_id => resolve_document_id(event, event_id),
|
548
|
+
:_index => resolve_index!(event, event_index),
|
548
549
|
routing_field_name => @routing ? event.sprintf(@routing) : nil
|
549
550
|
}
|
550
551
|
|
551
|
-
target_pipeline = resolve_pipeline(event)
|
552
|
+
target_pipeline = resolve_pipeline(event, event_pipeline)
|
552
553
|
# convention: empty string equates to not using a pipeline
|
553
554
|
# this is useful when using a field reference in the pipeline setting, e.g.
|
554
555
|
# elasticsearch {
|
@@ -559,7 +560,26 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
559
560
|
params
|
560
561
|
end
|
561
562
|
|
562
|
-
def
|
563
|
+
def resolve_document_id(event, event_id)
|
564
|
+
return event.sprintf(@document_id) if @document_id
|
565
|
+
return event_id || nil
|
566
|
+
end
|
567
|
+
private :resolve_document_id
|
568
|
+
|
569
|
+
def resolve_index!(event, event_index)
|
570
|
+
sprintf_index = @event_target.call(event)
|
571
|
+
raise IndexInterpolationError, sprintf_index if sprintf_index.match(/%{.*?}/) && dlq_on_failed_indexname_interpolation
|
572
|
+
# if it's not a data stream, sprintf_index is the @index with resolved placeholders.
|
573
|
+
# if is a data stream, sprintf_index could be either the name of a data stream or the value contained in
|
574
|
+
# @index without placeholders substitution. If event's metadata index is provided, it takes precedence
|
575
|
+
# on datastream name or whatever is returned by the event_target provider.
|
576
|
+
return event_index if @index == @default_index && event_index
|
577
|
+
return sprintf_index
|
578
|
+
end
|
579
|
+
private :resolve_index!
|
580
|
+
|
581
|
+
def resolve_pipeline(event, event_pipeline)
|
582
|
+
return event_pipeline if event_pipeline && !@pipeline
|
563
583
|
pipeline_template = @pipeline || event.get("[@metadata][target_ingest_pipeline]")&.to_s
|
564
584
|
pipeline_template && event.sprintf(pipeline_template)
|
565
585
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-elasticsearch'
|
3
|
-
s.version = '11.
|
3
|
+
s.version = '11.21.0'
|
4
4
|
s.licenses = ['apache-2.0']
|
5
5
|
s.summary = "Stores logs in Elasticsearch"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -271,6 +271,158 @@ describe LogStash::Outputs::ElasticSearch do
|
|
271
271
|
end
|
272
272
|
end
|
273
273
|
|
274
|
+
describe "with event integration metadata" do
|
275
|
+
let(:event_fields) {{}}
|
276
|
+
let(:event) { LogStash::Event.new(event_fields)}
|
277
|
+
|
278
|
+
context "when plugin's index is specified" do
|
279
|
+
let(:options) { super().merge("index" => "index_from_settings")}
|
280
|
+
|
281
|
+
context "when the event contains an integration metadata index" do
|
282
|
+
let(:event_fields) { super().merge({"@metadata" => {"_ingest_document" => {"index" => "meta-document-index"}}}) }
|
283
|
+
|
284
|
+
it "plugin's index is used" do
|
285
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "index_from_settings")
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
context "when the event doesn't contains an integration metadata index" do
|
290
|
+
it "plugin's index is used" do
|
291
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "index_from_settings")
|
292
|
+
end
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
context "when plugin's index is NOT specified" do
|
297
|
+
let(:options) { super().merge("index" => nil)}
|
298
|
+
|
299
|
+
context "when the event contains an integration metadata index" do
|
300
|
+
let(:event_fields) { super().merge({"@metadata" => {"_ingest_document" => {"index" => "meta-document-index"}}}) }
|
301
|
+
|
302
|
+
it "event's metadata index is used" do
|
303
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "meta-document-index")
|
304
|
+
end
|
305
|
+
|
306
|
+
context "when datastream settings are NOT configured" do
|
307
|
+
it "event's metadata index is used" do
|
308
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "meta-document-index")
|
309
|
+
end
|
310
|
+
end
|
311
|
+
|
312
|
+
context "when datastream settings are configured" do
|
313
|
+
let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
|
314
|
+
|
315
|
+
it "event's metadata index is used" do
|
316
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "meta-document-index")
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|
320
|
+
|
321
|
+
context "when the event DOESN'T contain integration metadata index" do
|
322
|
+
let(:default_index_resolved) { event.sprintf(subject.default_index) }
|
323
|
+
|
324
|
+
it "default index is used" do
|
325
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => default_index_resolved)
|
326
|
+
end
|
327
|
+
|
328
|
+
context "when datastream settings are NOT configured" do
|
329
|
+
it "default index is used" do
|
330
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => default_index_resolved)
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
context "when datastream settings are configured" do
|
335
|
+
let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
|
336
|
+
|
337
|
+
it "default index is used" do
|
338
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => default_index_resolved)
|
339
|
+
end
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
context "when plugin's document_id is specified" do
|
345
|
+
let(:options) { super().merge("document_id" => "id_from_settings")}
|
346
|
+
|
347
|
+
context "when the event contains an integration metadata document_id" do
|
348
|
+
let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"id" => "meta-document-id"}}}) }
|
349
|
+
|
350
|
+
it "plugin's document_id is used" do
|
351
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => "id_from_settings")
|
352
|
+
end
|
353
|
+
end
|
354
|
+
|
355
|
+
context "when the event DOESN'T contains an integration metadata document_id" do
|
356
|
+
it "plugin's document_id is used" do
|
357
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => "id_from_settings")
|
358
|
+
end
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
362
|
+
context "when plugin's document_id is NOT specified" do
|
363
|
+
let(:options) { super().merge("document_id" => nil)}
|
364
|
+
|
365
|
+
context "when the event contains an integration metadata document_id" do
|
366
|
+
let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"id" => "meta-document-id"}}}) }
|
367
|
+
|
368
|
+
it "event's metadata document_id is used" do
|
369
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => "meta-document-id")
|
370
|
+
end
|
371
|
+
end
|
372
|
+
|
373
|
+
context "when the event DOESN'T contains an integration metadata document_id" do
|
374
|
+
it "plugin's default id mechanism is used" do
|
375
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => nil)
|
376
|
+
end
|
377
|
+
end
|
378
|
+
end
|
379
|
+
|
380
|
+
context "when plugin's pipeline is specified" do
|
381
|
+
let(:options) { {"pipeline" => "pipeline_from_settings" } }
|
382
|
+
|
383
|
+
context "when the event contains an integration metadata pipeline" do
|
384
|
+
let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"pipeline" => "integration-pipeline"}}}) }
|
385
|
+
|
386
|
+
it "plugin's pipeline is used" do
|
387
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "pipeline_from_settings")
|
388
|
+
end
|
389
|
+
end
|
390
|
+
|
391
|
+
context "when the event DOESN'T contains an integration metadata pipeline" do
|
392
|
+
it "plugin's pipeline is used" do
|
393
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "pipeline_from_settings")
|
394
|
+
end
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
context "when plugin's pipeline is NOT specified" do
|
399
|
+
let(:options) { super().merge("pipeline" => nil)}
|
400
|
+
|
401
|
+
context "when the event contains an integration metadata pipeline" do
|
402
|
+
let(:metadata) { {"_ingest_document" => {"pipeline" => "integration-pipeline"}} }
|
403
|
+
let(:event) { LogStash::Event.new({"@metadata" => metadata}) }
|
404
|
+
|
405
|
+
it "event's metadata pipeline is used" do
|
406
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "integration-pipeline")
|
407
|
+
end
|
408
|
+
|
409
|
+
context "when also target_ingest_pipeline id defined" do
|
410
|
+
let(:metadata) { super().merge({"target_ingest_pipeline" => "meta-ingest-pipeline"}) }
|
411
|
+
|
412
|
+
it "then event's pipeline from _ingest_document is used" do
|
413
|
+
expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "integration-pipeline")
|
414
|
+
end
|
415
|
+
end
|
416
|
+
end
|
417
|
+
|
418
|
+
context "when the event DOESN'T contains an integration metadata pipeline" do
|
419
|
+
it "plugin's default pipeline mechanism is used" do
|
420
|
+
expect(subject.send(:event_action_tuple, event)[1]).to_not have_key(:pipeline)
|
421
|
+
end
|
422
|
+
end
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
274
426
|
describe "with auth" do
|
275
427
|
let(:user) { "myuser" }
|
276
428
|
let(:password) { ::LogStash::Util::Password.new("mypassword") }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 11.
|
4
|
+
version: 11.21.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-11-
|
11
|
+
date: 2023-11-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|