logstash-output-elasticsearch 11.12.4-java → 11.13.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a912267d383b51082d4c1b68e21d83dcb506433abc4f3f12bce0465eef84f439
4
- data.tar.gz: dc69623c94ad47986ed07f25818eeabfbfdd49532dfd9970f8c638dfe950f615
3
+ metadata.gz: d83bbddeedf7f5674416d431b0a54d7d939a7fd4a21853847f96c2ecf44659c8
4
+ data.tar.gz: 5d92c4dfd6e5843c7298021b98dcc86dcdfa3f5e8f474255c0358723a673d9cb
5
5
  SHA512:
6
- metadata.gz: 99568149dc8d313460046ca2e247c43e2e505e6649441a22a365e0d5484ba493d5d91c4eedfb5700fc447f64122555db3b92d6b59513a6b42e07dcc43dda3ada
7
- data.tar.gz: 56d2ce9c942f2dffbfc850f8feec0368dc2456ee08f7d7f46b83de94f9439fdd9047a1bae9231cc41caf45105a1abda0d1e60429a37059709f337e5ea8f8b6e4
6
+ metadata.gz: 47194a7711b93f6a1dcca191dc37a7c0bb14ddcf5591940520ad3e24e046df67850ec0fbf1c9ed769ff6706c7bd9b087bd44cd2d7b97be444bd7d8c5c48ad295
7
+ data.tar.gz: b7344d10ba9a8a09a5acb65348a4e7152407a3d98126843a8266112b67bc8e6a4859e7ab60e4a7d1bd448cde8d00d65c2e5dda9572eec175fc5cc2baab17441d
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 11.13.1
2
+ - Avoid crash by ensuring ILM settings are injected in the correct location depending on the default (or custom) template format, template_api setting and ES version [#1102](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1102)
3
+
4
+ ## 11.13.0
5
+ - add technology preview support for allowing events to individually encode a default pipeline with `[@metadata][target_ingest_pipeline]` (as part of a technology preview, this feature may change without notice) [#1113](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1113)
6
+
1
7
  ## 11.12.4
2
8
  - Changed the `manage_template` default value to `false` when data streams is enabled [#1111](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1111)
3
9
  - Added the `manage_template => false` as a valid data stream option
data/docs/index.asciidoc CHANGED
@@ -849,12 +849,11 @@ not also set this field. That will raise an error at startup
849
849
  ===== `pipeline`
850
850
 
851
851
  * Value type is <<string,string>>
852
- * Default value is `nil`
852
+ * There is no default value.
853
853
 
854
854
  Set which ingest pipeline you wish to execute for an event. You can also use
855
- event dependent configuration here like `pipeline =>
856
- "%{[@metadata][pipeline]}"`. The pipeline parameter won't be set if the value
857
- resolves to empty string ("").
855
+ event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`.
856
+ The pipeline parameter won't be set if the value resolves to empty string ("").
858
857
 
859
858
  [id="plugins-{type}s-{plugin}-pool_max"]
860
859
  ===== `pool_max`
@@ -46,15 +46,38 @@ module LogStash; module Outputs; class ElasticSearch
46
46
  # definition - remove any existing definition of 'template'
47
47
  template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8
48
48
  template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
49
- settings = template_settings(plugin, template)
49
+ settings = resolve_template_settings(plugin, template)
50
50
  if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias'])
51
51
  plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled")
52
52
  end
53
53
  settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
54
54
  end
55
55
 
56
- def self.template_settings(plugin, template)
57
- plugin.maximum_seen_major_version < 8 ? template['settings']: template['template']['settings']
56
+ def self.resolve_template_settings(plugin, template)
57
+ if template.key?('template')
58
+ plugin.logger.trace("Resolving ILM template settings: under 'template' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version)
59
+ composable_index_template_settings(template)
60
+ elsif template.key?('settings')
61
+ plugin.logger.trace("Resolving ILM template settings: under 'settings' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version)
62
+ legacy_index_template_settings(template)
63
+ else
64
+ template_endpoint = template_endpoint(plugin)
65
+ plugin.logger.trace("Resolving ILM template settings: template doesn't have 'settings' or 'template' fields, falling back to auto detection", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version, :template_endpoint => template_endpoint)
66
+ template_endpoint == INDEX_TEMPLATE_ENDPOINT ?
67
+ composable_index_template_settings(template) :
68
+ legacy_index_template_settings(template)
69
+ end
70
+ end
71
+
72
+ # Sets ['settings'] field to be compatible with _template API structure
73
+ def self.legacy_index_template_settings(template)
74
+ template['settings'] ||= {}
75
+ end
76
+
77
+ # Sets the ['template']['settings'] fields if not exist to be compatible with _index_template API structure
78
+ def self.composable_index_template_settings(template)
79
+ template['template'] ||= {}
80
+ template['template']['settings'] ||= {}
58
81
  end
59
82
 
60
83
  # Template name - if template_name set, use it
@@ -524,19 +524,22 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
524
524
  routing_field_name => @routing ? event.sprintf(@routing) : nil
525
525
  }
526
526
 
527
- if @pipeline
528
- value = event.sprintf(@pipeline)
529
- # convention: empty string equates to not using a pipeline
530
- # this is useful when using a field reference in the pipeline setting, e.g.
531
- # elasticsearch {
532
- # pipeline => "%{[@metadata][pipeline]}"
533
- # }
534
- params[:pipeline] = value unless value.empty?
535
- end
527
+ target_pipeline = resolve_pipeline(event)
528
+ # convention: empty string equates to not using a pipeline
529
+ # this is useful when using a field reference in the pipeline setting, e.g.
530
+ # elasticsearch {
531
+ # pipeline => "%{[@metadata][pipeline]}"
532
+ # }
533
+ params[:pipeline] = target_pipeline unless (target_pipeline.nil? || target_pipeline.empty?)
536
534
 
537
535
  params
538
536
  end
539
537
 
538
+ def resolve_pipeline(event)
539
+ pipeline_template = @pipeline || event.get("[@metadata][target_ingest_pipeline]")&.to_s
540
+ pipeline_template && event.sprintf(pipeline_template)
541
+ end
542
+
540
543
  @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
541
544
 
542
545
  @@plugins.each do |plugin|
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '11.12.4'
3
+ s.version = '11.13.1'
4
4
  s.licenses = ['apache-2.0']
5
5
  s.summary = "Stores logs in Elasticsearch"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -1,4 +1,4 @@
1
- require "logstash/devutils/rspec/spec_helper"
1
+ require_relative "../../../../spec/spec_helper"
2
2
  require "logstash/outputs/elasticsearch/template_manager"
3
3
 
4
4
  describe LogStash::Outputs::ElasticSearch::TemplateManager do
@@ -33,33 +33,85 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
33
33
  end
34
34
  end
35
35
 
36
- describe "index template with ilm settings" do
36
+ context "index template with ilm settings" do
37
37
  let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
38
38
  let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
39
39
 
40
- describe "in version 8+" do
41
- let(:file_path) { described_class.default_template_path(8) }
42
- let(:template) { described_class.read_template_file(file_path)}
40
+ describe "with custom template" do
43
41
 
44
- it "should update settings" do
45
- expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
46
- described_class.add_ilm_settings_to_template(plugin, template)
47
- expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
48
- expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
49
- expect(template.include?('settings')).to be_falsey
42
+ describe "in version 8+" do
43
+ let(:file_path) { described_class.default_template_path(8) }
44
+ let(:template) { described_class.read_template_file(file_path)}
45
+
46
+ it "should update settings" do
47
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
48
+ described_class.add_ilm_settings_to_template(plugin, template)
49
+ expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
50
+ expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
51
+ expect(template.include?('settings')).to be_falsey
52
+ end
53
+ end
54
+
55
+ describe "in version < 8" do
56
+ let(:file_path) { described_class.default_template_path(7) }
57
+ let(:template) { described_class.read_template_file(file_path)}
58
+
59
+ it "should update settings" do
60
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
61
+ described_class.add_ilm_settings_to_template(plugin, template)
62
+ expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
63
+ expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
64
+ expect(template.include?('template')).to be_falsey
65
+ end
50
66
  end
51
67
  end
52
68
 
53
- describe "in version < 8" do
54
- let(:file_path) { described_class.default_template_path(7) }
55
- let(:template) { described_class.read_template_file(file_path)}
69
+ context "resolve template setting" do
70
+ let(:plugin_settings) { super().merge({"template_api" => template_api}) }
71
+
72
+ describe "with composable template API" do
73
+ let(:template_api) { "composable" }
56
74
 
57
- it "should update settings" do
58
- expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
59
- described_class.add_ilm_settings_to_template(plugin, template)
60
- expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
61
- expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
62
- expect(template.include?('template')).to be_falsey
75
+ it 'resolves composable index template API compatible setting' do
76
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
77
+ template = {}
78
+ described_class.resolve_template_settings(plugin, template)
79
+ expect(template["template"]["settings"]).not_to eq(nil)
80
+ end
81
+ end
82
+
83
+ describe "with legacy template API" do
84
+ let(:template_api) { "legacy" }
85
+
86
+ it 'resolves legacy index template API compatible setting' do
87
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
88
+ template = {}
89
+ described_class.resolve_template_settings(plugin, template)
90
+ expect(template["settings"]).not_to eq(nil)
91
+ end
92
+ end
93
+
94
+ describe "with `template_api => 'auto'`" do
95
+ let(:template_api) { "auto" }
96
+
97
+ describe "with ES < 8 versions" do
98
+
99
+ it 'resolves legacy index template API compatible setting' do
100
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
101
+ template = {}
102
+ described_class.resolve_template_settings(plugin, template)
103
+ expect(template["settings"]).not_to eq(nil)
104
+ end
105
+ end
106
+
107
+ describe "with ES >= 8 versions" do
108
+ it 'resolves composable index template API compatible setting' do
109
+ expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
110
+ template = {}
111
+ described_class.resolve_template_settings(plugin, template)
112
+ expect(template["template"]["settings"]).not_to eq(nil)
113
+ end
114
+ end
63
115
  end
64
116
  end
65
117
  end
@@ -590,7 +590,7 @@ describe LogStash::Outputs::ElasticSearch do
590
590
 
591
591
  let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
592
592
 
593
- it "should interpolate the pipeline value and set it" do
593
+ it "interpolate the pipeline value and set it" do
594
594
  expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
595
595
  end
596
596
  end
@@ -600,7 +600,66 @@ describe LogStash::Outputs::ElasticSearch do
600
600
 
601
601
  let(:event) { LogStash::Event.new("pipeline" => "") }
602
602
 
603
- it "should interpolate the pipeline value but not set it because it is empty" do
603
+ it "interpolates the pipeline value but not set it because it is empty" do
604
+ expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
605
+ end
606
+ end
607
+
608
+ context "with both pipeline and target_ingest_pipeline" do
609
+ let(:options) { {"pipeline" => "%{pipeline}" } }
610
+ let(:event) { LogStash::Event.new({"pipeline" => "my-ingest-pipeline", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
611
+
612
+ it "interpolates the plugin's pipeline value" do
613
+ expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
614
+ end
615
+
616
+ context "when the plugin's `pipeline` is constant" do
617
+ let(:options) { super().merge("pipeline" => "my-constant-pipeline") }
618
+ it "uses plugin's pipeline value" do
619
+ expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-constant-pipeline")
620
+ end
621
+ end
622
+
623
+ context "when the plugin's `pipeline` includes an unresolvable sprintf placeholder" do
624
+ let(:options) { super().merge("pipeline" => "reference-%{unset}-field") }
625
+ it "does not use the target_ingest_pipeline" do
626
+ # when sprintf doesn't resolve a placeholder, the behaviour of our `pipeline` is UNSPECIFIED.
627
+ # here we only validate that the presence of the magic field does not
628
+ # override an explicitly-configured pipeline.
629
+ expect(subject.send(:event_action_tuple, event)[1]).to_not include(:pipeline => "my-ingest-pipeline")
630
+ end
631
+ end
632
+ end
633
+
634
+ context "with empty pipeline and target_ingest_pipeline" do
635
+ let(:options) { {"pipeline" => "%{pipeline}" } }
636
+ let(:event) { LogStash::Event.new({"pipeline" => "", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
637
+
638
+ it "interpolates the pipeline value but not set it because pipeline is empty" do
639
+ expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
640
+ end
641
+ end
642
+
643
+ context "with target_ingest_pipeline" do
644
+ let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"target_ingest_pipeline" => "meta-ingest-pipeline"}}) }
645
+
646
+ it "interpolates the target_ingest_pipeline value and set it" do
647
+ expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "meta-ingest-pipeline")
648
+ end
649
+ end
650
+
651
+ context "with empty target_ingest_pipeline" do
652
+ let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"host" => "elastic"}}) }
653
+
654
+ it "does not set pipeline" do
655
+ expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
656
+ end
657
+ end
658
+
659
+ context "with empty pipeline and empty target_ingest_pipeline" do
660
+ let(:event) { LogStash::Event.new }
661
+
662
+ it "does not set pipeline" do
604
663
  expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
605
664
  end
606
665
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.12.4
4
+ version: 11.13.1
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-02-01 00:00:00.000000000 Z
11
+ date: 2023-02-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement