logstash-filter-elasticsearch 4.2.0 → 4.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -0
  3. data/docs/index.asciidoc +132 -6
  4. data/lib/logstash/filters/elasticsearch/client.rb +16 -2
  5. data/lib/logstash/filters/elasticsearch/dsl_executor.rb +140 -0
  6. data/lib/logstash/filters/elasticsearch/esql_executor.rb +178 -0
  7. data/lib/logstash/filters/elasticsearch.rb +105 -129
  8. data/logstash-filter-elasticsearch.gemspec +1 -1
  9. data/spec/filters/elasticsearch_dsl_spec.rb +372 -0
  10. data/spec/filters/elasticsearch_esql_spec.rb +211 -0
  11. data/spec/filters/elasticsearch_spec.rb +149 -332
  12. data/spec/filters/integration/elasticsearch_esql_spec.rb +167 -0
  13. metadata +20 -15
data/spec/filters/integration/elasticsearch_esql_spec.rb ADDED
@@ -0,0 +1,167 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/filters/elasticsearch"
4
+ require "elasticsearch"
5
+ require_relative "../../../spec/es_helper"
6
+
7
+ describe LogStash::Filters::Elasticsearch, integration: true do
8
+
9
+ ELASTIC_SECURITY_ENABLED = ENV['ELASTIC_SECURITY_ENABLED'].eql? 'true'
10
+ SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
11
+ ES_HOSTS = ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"]
12
+ CA_PATH = File.expand_path('../fixtures/test_certs/ca.crt', File.dirname(__FILE__))
13
+
14
+ let(:plugin) { described_class.new(config) }
15
+ let(:es_index) { "es-filter-plugin-esql-integration-#{rand(1000)}" }
16
+ let(:test_documents) do
17
+ [
18
+ { "message" => "test message 1", "type" => "a", "count" => 1 },
19
+ { "message" => "test message 2", "type" => "a", "count" => 2 },
20
+ { "message" => "test message 3", "type" => "b", "count" => 3 },
21
+ { "message" => "test message 4", "type" => "b", "count" => 4 },
22
+ { "message" => "test message 5", "type" => "c", "count" => 5 },
23
+ { "message" => "odd test message", "type" => "t" }
24
+ ]
25
+ end
26
+
27
+ let(:base_config) do
28
+ {
29
+ "query_type" => "esql",
30
+ "hosts" => ES_HOSTS,
31
+ "ssl_enabled" => SECURE_INTEGRATION
32
+ }
33
+ end
34
+
35
+ let(:credentials) do
36
+ if SECURE_INTEGRATION
37
+ { 'user' => 'tests', 'password' => 'Tests123' }
38
+ else
39
+ { 'user' => 'elastic', 'password' => ENV['ELASTIC_PASSWORD'] }
40
+ end
41
+ end
42
+
43
+ let(:config) do
44
+ config = ELASTIC_SECURITY_ENABLED ? base_config.merge(credentials) : base_config
45
+ config = { 'ssl_certificate_authorities' => CA_PATH }.merge(config) if SECURE_INTEGRATION
46
+ config
47
+ end
48
+
49
+ let(:event) { LogStash::Event.new({}) }
50
+
51
+ def es_client
52
+ @es_client ||= begin
53
+ user = SECURE_INTEGRATION ? 'tests' : 'elastic'
54
+ password = SECURE_INTEGRATION ? 'Tests123' : ENV['ELASTIC_PASSWORD']
55
+
56
+ es_client_config = { hosts: ES_HOSTS }
57
+ es_client_config = es_client_config.merge({ user: user, password: password }) if ELASTIC_SECURITY_ENABLED || SECURE_INTEGRATION
58
+ es_client_config = es_client_config.merge({ transport_options: { ssl: { ca_path: CA_PATH, verify: false }}}) if SECURE_INTEGRATION
59
+
60
+ Elasticsearch::Client.new(es_client_config)
61
+ end
62
+ end
63
+
64
+ before(:all) do
65
+ is_ls_with_esql_supported_client = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
66
+ # Skip tests if an ES version doesn't support ES|QL
67
+ skip "LS version does not have ES client which supports ES|QL" unless is_ls_with_esql_supported_client
68
+
69
+ es_version_info = es_client.info["version"]
70
+ es_gem_version = Gem::Version.create(es_version_info["number"])
71
+ skip "ES version does not support ES|QL" if es_gem_version.nil? || es_gem_version < Gem::Version.create(LogStash::Filters::Elasticsearch::ES_ESQL_SUPPORT_VERSION)
72
+ end
73
+
74
+ before(:each) do
75
+ # Create index with test documents
76
+ es_client.indices.create(index: es_index, body: {}) unless es_client.indices.exists?(index: es_index)
77
+
78
+ test_documents.each do |doc|
79
+ es_client.index(index: es_index, body: doc, refresh: true)
80
+ end
81
+ end
82
+
83
+ after(:each) do
84
+ es_client.indices.delete(index: es_index) if es_client.indices.exists?(index: es_index)
85
+ end
86
+
87
+ describe "run ES|QL queries" do
88
+
89
+ before do
90
+ stub_const("LOGSTASH_VERSION", LogStash::Filters::Elasticsearch::LS_ESQL_SUPPORT_VERSION)
91
+ end
92
+
93
+ before(:each) do
94
+ plugin.register
95
+ end
96
+
97
+ shared_examples "ESQL query execution" do |expected_count, fields|
98
+ it "processes the event" do
99
+ plugin.filter(event)
100
+ expect(event.get("[@metadata][total_values]")).to eq(expected_count)
101
+ fields&.each do | field |
102
+ expect(event.get(field)).not_to be(nil)
103
+ end
104
+ end
105
+ end
106
+
107
+ describe "with simple FROM query with LIMIT" do
108
+ let(:config) do
109
+ super().merge("query" => "FROM #{es_index} | LIMIT 99")
110
+ end
111
+
112
+ include_examples "ESQL query execution", 6
113
+ end
114
+
115
+ describe "with simple FROM and WHERE query combinations" do
116
+ let(:config) do
117
+ super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99")
118
+ end
119
+
120
+ include_examples "ESQL query execution", 2
121
+ end
122
+
123
+ describe "with query params" do
124
+ let(:config) do
125
+ super().merge("query" => "FROM #{es_index} | WHERE type==?type", "query_params" => { "type" => "b" })
126
+ end
127
+
128
+ include_examples "ESQL query execution", 2
129
+ end
130
+
131
+ describe "when invalid query used" do
132
+ let(:config) do
133
+ super().merge("query" => "FROM undefined index | LIMIT 1")
134
+ end
135
+
136
+ it "tags on failure" do
137
+ plugin.filter(event)
138
+ expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
139
+ end
140
+ end
141
+
142
+ describe "when field enrichment requested" do
143
+ let(:config) do
144
+ super().merge("query" => "FROM #{es_index} | WHERE type==\"b\" | LIMIT 99")
145
+ end
146
+
147
+ include_examples "ESQL query execution", 2, %w[message count]
148
+ end
149
+
150
+ describe "when non-exist field value appear" do
151
+ let(:config) do
152
+ super().merge("query" => "FROM #{es_index}", "target" => "target_field")
153
+ end
154
+
155
+ it "processes the event" do
156
+ plugin.filter(event)
157
+ expect(event.get("[@metadata][total_values]")).to eq(6)
158
+ expect(event.get("target_field").size).to eq(6)
159
+ values = event.get("target_field")
160
+ counts = values.count { |entry| entry.key?("count") }
161
+ messages = values.count { |entry| entry.key?("message") }
162
+ expect(counts).to eq(5)
163
+ expect(messages).to eq(6)
164
+ end
165
+ end
166
+ end
167
+ end
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.2.0
4
+ version: 4.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2025-05-07 00:00:00.000000000 Z
10
+ date: 2025-09-23 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
13
+ name: logstash-core-plugin-api
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
16
  - - ">="
@@ -19,7 +19,6 @@ dependencies:
19
19
  - - "<="
20
20
  - !ruby/object:Gem::Version
21
21
  version: '2.99'
22
- name: logstash-core-plugin-api
23
22
  type: :runtime
24
23
  prerelease: false
25
24
  version_requirements: !ruby/object:Gem::Requirement
@@ -31,6 +30,7 @@ dependencies:
31
30
  - !ruby/object:Gem::Version
32
31
  version: '2.99'
33
32
  - !ruby/object:Gem::Dependency
33
+ name: elasticsearch
34
34
  requirement: !ruby/object:Gem::Requirement
35
35
  requirements:
36
36
  - - ">="
@@ -39,7 +39,6 @@ dependencies:
39
39
  - - "<"
40
40
  - !ruby/object:Gem::Version
41
41
  version: '9'
42
- name: elasticsearch
43
42
  type: :runtime
44
43
  prerelease: false
45
44
  version_requirements: !ruby/object:Gem::Requirement
@@ -51,12 +50,12 @@ dependencies:
51
50
  - !ruby/object:Gem::Version
52
51
  version: '9'
53
52
  - !ruby/object:Gem::Dependency
53
+ name: manticore
54
54
  requirement: !ruby/object:Gem::Requirement
55
55
  requirements:
56
56
  - - ">="
57
57
  - !ruby/object:Gem::Version
58
58
  version: 0.7.1
59
- name: manticore
60
59
  type: :runtime
61
60
  prerelease: false
62
61
  version_requirements: !ruby/object:Gem::Requirement
@@ -65,12 +64,12 @@ dependencies:
65
64
  - !ruby/object:Gem::Version
66
65
  version: 0.7.1
67
66
  - !ruby/object:Gem::Dependency
67
+ name: logstash-mixin-ecs_compatibility_support
68
68
  requirement: !ruby/object:Gem::Requirement
69
69
  requirements:
70
70
  - - "~>"
71
71
  - !ruby/object:Gem::Version
72
72
  version: '1.3'
73
- name: logstash-mixin-ecs_compatibility_support
74
73
  type: :runtime
75
74
  prerelease: false
76
75
  version_requirements: !ruby/object:Gem::Requirement
@@ -79,12 +78,12 @@ dependencies:
79
78
  - !ruby/object:Gem::Version
80
79
  version: '1.3'
81
80
  - !ruby/object:Gem::Dependency
81
+ name: logstash-mixin-ca_trusted_fingerprint_support
82
82
  requirement: !ruby/object:Gem::Requirement
83
83
  requirements:
84
84
  - - "~>"
85
85
  - !ruby/object:Gem::Version
86
86
  version: '1.0'
87
- name: logstash-mixin-ca_trusted_fingerprint_support
88
87
  type: :runtime
89
88
  prerelease: false
90
89
  version_requirements: !ruby/object:Gem::Requirement
@@ -93,12 +92,12 @@ dependencies:
93
92
  - !ruby/object:Gem::Version
94
93
  version: '1.0'
95
94
  - !ruby/object:Gem::Dependency
95
+ name: logstash-mixin-validator_support
96
96
  requirement: !ruby/object:Gem::Requirement
97
97
  requirements:
98
98
  - - "~>"
99
99
  - !ruby/object:Gem::Version
100
100
  version: '1.0'
101
- name: logstash-mixin-validator_support
102
101
  type: :runtime
103
102
  prerelease: false
104
103
  version_requirements: !ruby/object:Gem::Requirement
@@ -107,12 +106,12 @@ dependencies:
107
106
  - !ruby/object:Gem::Version
108
107
  version: '1.0'
109
108
  - !ruby/object:Gem::Dependency
109
+ name: cabin
110
110
  requirement: !ruby/object:Gem::Requirement
111
111
  requirements:
112
112
  - - "~>"
113
113
  - !ruby/object:Gem::Version
114
114
  version: '0.6'
115
- name: cabin
116
115
  type: :development
117
116
  prerelease: false
118
117
  version_requirements: !ruby/object:Gem::Requirement
@@ -121,12 +120,12 @@ dependencies:
121
120
  - !ruby/object:Gem::Version
122
121
  version: '0.6'
123
122
  - !ruby/object:Gem::Dependency
123
+ name: webrick
124
124
  requirement: !ruby/object:Gem::Requirement
125
125
  requirements:
126
126
  - - ">="
127
127
  - !ruby/object:Gem::Version
128
128
  version: '0'
129
- name: webrick
130
129
  type: :development
131
130
  prerelease: false
132
131
  version_requirements: !ruby/object:Gem::Requirement
@@ -135,12 +134,12 @@ dependencies:
135
134
  - !ruby/object:Gem::Version
136
135
  version: '0'
137
136
  - !ruby/object:Gem::Dependency
137
+ name: logstash-devutils
138
138
  requirement: !ruby/object:Gem::Requirement
139
139
  requirements:
140
140
  - - ">="
141
141
  - !ruby/object:Gem::Version
142
142
  version: '0'
143
- name: logstash-devutils
144
143
  type: :development
145
144
  prerelease: false
146
145
  version_requirements: !ruby/object:Gem::Requirement
@@ -165,9 +164,13 @@ files:
165
164
  - docs/index.asciidoc
166
165
  - lib/logstash/filters/elasticsearch.rb
167
166
  - lib/logstash/filters/elasticsearch/client.rb
167
+ - lib/logstash/filters/elasticsearch/dsl_executor.rb
168
+ - lib/logstash/filters/elasticsearch/esql_executor.rb
168
169
  - lib/logstash/filters/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
169
170
  - logstash-filter-elasticsearch.gemspec
170
171
  - spec/es_helper.rb
172
+ - spec/filters/elasticsearch_dsl_spec.rb
173
+ - spec/filters/elasticsearch_esql_spec.rb
171
174
  - spec/filters/elasticsearch_spec.rb
172
175
  - spec/filters/elasticsearch_ssl_spec.rb
173
176
  - spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
@@ -193,6 +196,7 @@ files:
193
196
  - spec/filters/fixtures/test_certs/ls.crt
194
197
  - spec/filters/fixtures/test_certs/ls.der.sha256
195
198
  - spec/filters/fixtures/test_certs/ls.key
199
+ - spec/filters/integration/elasticsearch_esql_spec.rb
196
200
  - spec/filters/integration/elasticsearch_spec.rb
197
201
  homepage: https://elastic.co/logstash
198
202
  licenses:
@@ -200,7 +204,6 @@ licenses:
200
204
  metadata:
201
205
  logstash_plugin: 'true'
202
206
  logstash_group: filter
203
- post_install_message:
204
207
  rdoc_options: []
205
208
  require_paths:
206
209
  - lib
@@ -215,12 +218,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
215
218
  - !ruby/object:Gem::Version
216
219
  version: '0'
217
220
  requirements: []
218
- rubygems_version: 3.3.26
219
- signing_key:
221
+ rubygems_version: 3.6.3
220
222
  specification_version: 4
221
223
  summary: Copies fields from previous log events in Elasticsearch to current events
222
224
  test_files:
223
225
  - spec/es_helper.rb
226
+ - spec/filters/elasticsearch_dsl_spec.rb
227
+ - spec/filters/elasticsearch_esql_spec.rb
224
228
  - spec/filters/elasticsearch_spec.rb
225
229
  - spec/filters/elasticsearch_ssl_spec.rb
226
230
  - spec/filters/fixtures/elasticsearch_7.x_hits_total_as_object.json
@@ -246,4 +250,5 @@ test_files:
246
250
  - spec/filters/fixtures/test_certs/ls.crt
247
251
  - spec/filters/fixtures/test_certs/ls.der.sha256
248
252
  - spec/filters/fixtures/test_certs/ls.key
253
+ - spec/filters/integration/elasticsearch_esql_spec.rb
249
254
  - spec/filters/integration/elasticsearch_spec.rb