logstash-filter-opensearch-manticore 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,431 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/plugin"
4
+ require "logstash/filters/opensearch"
5
+ require "logstash/json"
6
+
7
+ describe LogStash::Filters::OpenSearch do
8
+
9
+ context "registration" do
10
+
11
+ let(:plugin) { LogStash::Plugin.lookup("filter", "opensearch").new({}) }
12
+ before do
13
+ allow(plugin).to receive(:test_connection!)
14
+ end
15
+
16
+ it "should not raise an exception" do
17
+ expect {plugin.register}.to_not raise_error
18
+ end
19
+ end
20
+
21
+ describe "data fetch" do
22
+ let(:config) do
23
+ {
24
+ "hosts" => ["localhost:9200"],
25
+ "query" => "response: 404",
26
+ "fields" => { "response" => "code" },
27
+ "docinfo_fields" => { "_index" => "opensearch_index" },
28
+ "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
29
+ }
30
+ end
31
+ let(:plugin) { described_class.new(config) }
32
+ let(:event) { LogStash::Event.new({}) }
33
+
34
+ let(:response) do
35
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
36
+ end
37
+
38
+ let(:client) { double(:client) }
39
+
40
+ before(:each) do
41
+ allow(LogStash::Filters::OpenSearchClient).to receive(:new).and_return(client)
42
+ allow(client).to receive(:search).and_return(response)
43
+ allow(plugin).to receive(:test_connection!)
44
+ plugin.register
45
+ end
46
+
47
+ after(:each) do
48
+ Thread.current[:filter_opensearch_client] = nil
49
+ end
50
+
51
+ # Since the OpenSearch Ruby client is not thread safe
52
+ # and under high load we can get error with the connection pool
53
+ # we have decided to create a new instance per worker thread which
54
+ # will be lazy created on the first call to `#filter`
55
+ #
56
+ # I am adding a simple test case for future changes
57
+ it "uses a different connection object per thread wait" do
58
+ expect(plugin.clients_pool.size).to eq(0)
59
+
60
+ Thread.new { plugin.filter(event) }.join
61
+ Thread.new { plugin.filter(event) }.join
62
+
63
+ expect(plugin.clients_pool.size).to eq(2)
64
+ end
65
+
66
+ it "should enhance the current event with new data" do
67
+ plugin.filter(event)
68
+ expect(event.get("code")).to eq(404)
69
+ expect(event.get("opensearch_index")).to eq("logstash-2014.08.26")
70
+ expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
71
+ end
72
+
73
+ it "should receive all necessary params to perform the search" do
74
+ expect(client).to receive(:search).with({:q=>"response: 404", :size=>1, :index=>"", :sort=>"@timestamp:desc"})
75
+ plugin.filter(event)
76
+ end
77
+
78
+ context "when asking to hit specific index" do
79
+
80
+ let(:config) do
81
+ {
82
+ "index" => "foo*",
83
+ "hosts" => ["localhost:9200"],
84
+ "query" => "response: 404",
85
+ "fields" => { "response" => "code" }
86
+ }
87
+ end
88
+
89
+ it "should receive all necessary params to perform the search" do
90
+ expect(client).to receive(:search).with({:q=>"response: 404", :size=>1, :index=>"foo*", :sort=>"@timestamp:desc"})
91
+ plugin.filter(event)
92
+ end
93
+ end
94
+
95
+ context "when asking for more than one result" do
96
+
97
+ let(:config) do
98
+ {
99
+ "hosts" => ["localhost:9200"],
100
+ "query" => "response: 404",
101
+ "fields" => { "response" => "code" },
102
+ "result_size" => 10
103
+ }
104
+ end
105
+
106
+ let(:response) do
107
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
108
+ end
109
+
110
+ it "should enhance the current event with new data" do
111
+ plugin.filter(event)
112
+ expect(event.get("code")).to eq([404]*10)
113
+ end
114
+ end
115
+
116
+ context 'when OpenSearch 7.x gives us a totals object instead of an integer' do
117
+ let(:config) do
118
+ {
119
+ "hosts" => ["localhost:9200"],
120
+ "query" => "response: 404",
121
+ "fields" => { "response" => "code" },
122
+ "result_size" => 10
123
+ }
124
+ end
125
+
126
+ let(:response) do
127
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "opensearch_7.x_hits_total_as_object.json")))
128
+ end
129
+
130
+ it "should enhance the current event with new data" do
131
+ plugin.filter(event)
132
+ expect(event.get("[@metadata][total_hits]")).to eq(13476)
133
+ end
134
+ end
135
+
136
+ context "if something wrong happen during connection" do
137
+
138
+ before(:each) do
139
+ allow(LogStash::Filters::OpenSearchClient).to receive(:new).and_return(client)
140
+ allow(client).to receive(:search).and_raise("connection exception")
141
+ plugin.register
142
+ end
143
+
144
+ it "tag the event as something happened, but still deliver it" do
145
+ expect(plugin.logger).to receive(:warn)
146
+ plugin.filter(event)
147
+ expect(event.to_hash["tags"]).to include("_opensearch_lookup_failure")
148
+ end
149
+ end
150
+
151
+ # Tagging test for positive results
152
+ context "Tagging should occur if query returns results" do
153
+ let(:config) do
154
+ {
155
+ "index" => "foo*",
156
+ "hosts" => ["localhost:9200"],
157
+ "query" => "response: 404",
158
+ "add_tag" => ["tagged"]
159
+ }
160
+ end
161
+
162
+ let(:response) do
163
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
164
+ end
165
+
166
+ it "should tag the current event if results returned" do
167
+ plugin.filter(event)
168
+ expect(event.to_hash["tags"]).to include("tagged")
169
+ end
170
+ end
171
+
172
+ context "an aggregation search with size 0 that matches" do
173
+ let(:config) do
174
+ {
175
+ "index" => "foo*",
176
+ "hosts" => ["localhost:9200"],
177
+ "query" => "response: 404",
178
+ "add_tag" => ["tagged"],
179
+ "result_size" => 0,
180
+ "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
181
+ }
182
+ end
183
+
184
+ let(:response) do
185
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_size0_agg.json")))
186
+ end
187
+
188
+ it "should tag the current event" do
189
+ plugin.filter(event)
190
+ expect(event.get("tags")).to include("tagged")
191
+ expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
192
+ end
193
+ end
194
+
195
+ # Tagging test for negative results
196
+ context "Tagging should not occur if query has no results" do
197
+ let(:config) do
198
+ {
199
+ "index" => "foo*",
200
+ "hosts" => ["localhost:9200"],
201
+ "query" => "response: 404",
202
+ "add_tag" => ["tagged"]
203
+ }
204
+ end
205
+
206
+ let(:response) do
207
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
208
+ end
209
+
210
+ it "should not tag the current event" do
211
+ plugin.filter(event)
212
+ expect(event.to_hash["tags"]).to_not include("tagged")
213
+ end
214
+ end
215
+ context "testing a simple query template" do
216
+ let(:config) do
217
+ {
218
+ "hosts" => ["localhost:9200"],
219
+ "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
220
+ "fields" => { "response" => "code" },
221
+ "result_size" => 1
222
+ }
223
+ end
224
+
225
+ let(:response) do
226
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
227
+ end
228
+
229
+ it "should enhance the current event with new data" do
230
+ plugin.filter(event)
231
+ expect(event.get("code")).to eq(404)
232
+ end
233
+
234
+ end
235
+
236
+ context "testing a simple index substitution" do
237
+ let(:event) {
238
+ LogStash::Event.new(
239
+ {
240
+ "subst_field" => "subst_value"
241
+ }
242
+ )
243
+ }
244
+ let(:config) do
245
+ {
246
+ "index" => "foo_%{subst_field}*",
247
+ "hosts" => ["localhost:9200"],
248
+ "query" => "response: 404",
249
+ "fields" => { "response" => "code" }
250
+ }
251
+ end
252
+
253
+ it "should receive substituted index name" do
254
+ expect(client).to receive(:search).with({:q => "response: 404", :size => 1, :index => "foo_subst_value*", :sort => "@timestamp:desc"})
255
+ plugin.filter(event)
256
+ end
257
+ end
258
+
259
+ context "if query result errored but no exception is thrown" do
260
+ let(:response) do
261
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
262
+ end
263
+
264
+ before(:each) do
265
+ allow(LogStash::Filters::OpenSearchClient).to receive(:new).and_return(client)
266
+ allow(client).to receive(:search).and_return(response)
267
+ plugin.register
268
+ end
269
+
270
+ it "tag the event as something happened, but still deliver it" do
271
+ expect(plugin.logger).to receive(:warn)
272
+ plugin.filter(event)
273
+ expect(event.to_hash["tags"]).to include("_opensearch_lookup_failure")
274
+ end
275
+ end
276
+
277
+ context "if query is on nested field" do
278
+ let(:config) do
279
+ {
280
+ "hosts" => ["localhost:9200"],
281
+ "query" => "response: 404",
282
+ "fields" => [ ["[geoip][ip]", "ip_address"] ]
283
+ }
284
+ end
285
+
286
+ it "should enhance the current event with new data" do
287
+ plugin.filter(event)
288
+ expect(event.get("ip_address")).to eq("66.249.73.185")
289
+ end
290
+
291
+ end
292
+ end
293
+
294
+ describe "client" do
295
+ let(:config) do
296
+ {
297
+ "query" => "response: unknown"
298
+ }
299
+ end
300
+ let(:plugin) { described_class.new(config) }
301
+ let(:event) { LogStash::Event.new({}) }
302
+
303
+ before(:each) do
304
+ allow(plugin).to receive(:test_connection!)
305
+ end
306
+
307
+ after(:each) do
308
+ Thread.current[:filter_opensearch_client] = nil
309
+ end
310
+
311
+ describe "cloud.id" do
312
+ let(:valid_cloud_id) do
313
+ 'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
314
+ end
315
+
316
+ let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
317
+
318
+ it "should set host(s)" do
319
+ plugin.register
320
+ client = plugin.send(:get_client).client
321
+ expect( client.transport.hosts ).to eql [{
322
+ :scheme => "https",
323
+ :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
324
+ :port => 9243,
325
+ :path => "",
326
+ :protocol => "https"
327
+ }]
328
+ end
329
+
330
+ context 'invalid' do
331
+ let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
332
+
333
+ it "should fail" do
334
+ expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
335
+ end
336
+ end
337
+
338
+ context 'hosts also set' do
339
+ let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
340
+
341
+ it "should fail" do
342
+ expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
343
+ end
344
+ end
345
+ end if LOGSTASH_VERSION > '6.0'
346
+
347
+ describe "cloud.auth" do
348
+ let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
349
+
350
+ it "should set authorization" do
351
+ plugin.register
352
+ client = plugin.send(:get_client).client
353
+ auth_header = client.transport.options[:transport_options][:headers][:Authorization]
354
+
355
+ expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
356
+ end
357
+
358
+ context 'invalid' do
359
+ let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
360
+
361
+ it "should fail" do
362
+ expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
363
+ end
364
+ end
365
+
366
+ context 'user also set' do
367
+ let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
368
+
369
+ it "should fail" do
370
+ expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
371
+ end
372
+ end
373
+ end if LOGSTASH_VERSION > '6.0'
374
+
375
+ describe "api_key" do
376
+ context "without ssl" do
377
+ let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
378
+
379
+ it "should fail" do
380
+ expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
381
+ end
382
+ end
383
+
384
+ context "with ssl" do
385
+ let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
386
+
387
+ it "should set authorization" do
388
+ plugin.register
389
+ client = plugin.send(:get_client).client
390
+ auth_header = client.transport.options[:transport_options][:headers][:Authorization]
391
+
392
+ expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
393
+ end
394
+
395
+ context 'user also set' do
396
+ let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
397
+
398
+ it "should fail" do
399
+ expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
400
+ end
401
+ end
402
+ end
403
+ end if LOGSTASH_VERSION > '6.0'
404
+ end
405
+
406
+ describe "query template" do
407
+ let(:config) do
408
+ {
409
+ "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template_unicode.json"),
410
+ }
411
+ end
412
+
413
+ let(:plugin) { described_class.new(config) }
414
+
415
+ let(:client) { double(:client) }
416
+
417
+ before(:each) do
418
+ allow(LogStash::Filters::OpenSearchClient).to receive(:new).and_return(client)
419
+ allow(plugin).to receive(:test_connection!)
420
+ plugin.register
421
+ end
422
+
423
+ it "should read and send non-ascii query" do
424
+ expect(client).to receive(:search).with(
425
+ :body => { "query" => { "terms" => { "lock" => [ "잠금", "uzávěr" ] } } },
426
+ :index => "")
427
+
428
+ plugin.filter(LogStash::Event.new)
429
+ end
430
+ end
431
+ end
@@ -0,0 +1,43 @@
1
+ module OpenSearchHelper
2
+ def self.get_host_port
3
+ if ENV["INTEGRATION"] == "true"
4
+ "opensearch:9200"
5
+ else
6
+ "localhost:9200"
7
+ end
8
+ end
9
+
10
+ def self.get_client
11
+ OpenSearch::Client.new(:hosts => [get_host_port])
12
+ end
13
+
14
+ def self.doc_type
15
+ if OpenSearchHelper.opensearch_version_satisfies?(">=8")
16
+ nil
17
+ elsif OpenSearchHelper.opensearch_version_satisfies?(">=7")
18
+ "_doc"
19
+ else
20
+ "doc"
21
+ end
22
+ end
23
+
24
+ def self.index_doc(opensearch, params)
25
+ type = doc_type
26
+ params[:type] = doc_type unless type.nil?
27
+ opensearch.index(params)
28
+ end
29
+
30
+ def self.opensearch_version
31
+ ENV['OPENSEARCH_VERSION']
32
+ end
33
+
34
+ def self.opensearch_version_satisfies?(*requirement)
35
+ opensearch_version = RSpec.configuration.filter[:opensearch_version] || ENV['OPENSEARCH_VERSION']
36
+ if opensearch_version.nil?
37
+ puts "Info: OPENSEARCH_VERSION or 'opensearch_version' tag wasn't set. Returning false to all `opensearch_version_satisfies?` call."
38
+ return false
39
+ end
40
+ opensearch_release_version = Gem::Version.new(opensearch_version).release
41
+ Gem::Requirement.new(requirement).satisfied_by?(opensearch_release_version)
42
+ end
43
+ end
metadata ADDED
@@ -0,0 +1,139 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-filter-opensearch-manticore
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Anton Klyba
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2023-05-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash-core-plugin-api
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '1.60'
20
+ - - "<="
21
+ - !ruby/object:Gem::Version
22
+ version: '2.99'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '1.60'
30
+ - - "<="
31
+ - !ruby/object:Gem::Version
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ name: opensearch-ruby
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ">="
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ name: manticore
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '0.6'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '0.6'
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ description: This gem is a Logstash plugin required to be installed on top of the
76
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
77
+ gem is not a stand-alone program
78
+ email: anarhyst266@gmail.com
79
+ executables: []
80
+ extensions: []
81
+ extra_rdoc_files: []
82
+ files:
83
+ - CHANGELOG.md
84
+ - CONTRIBUTORS
85
+ - Gemfile
86
+ - LICENSE
87
+ - NOTICE.TXT
88
+ - README.md
89
+ - docs/index.asciidoc
90
+ - lib/logstash/filters/opensearch.rb
91
+ - lib/logstash/filters/opensearch/patches/_opensearch_transport_connections_selector.rb
92
+ - lib/logstash/filters/opensearch/patches/_opensearch_transport_http_manticore.rb
93
+ - logstash-filter-opensearch-manticore.gemspec
94
+ - spec/filters/fixtures/opensearch_7.x_hits_total_as_object.json
95
+ - spec/filters/fixtures/query_template.json
96
+ - spec/filters/fixtures/query_template_unicode.json
97
+ - spec/filters/fixtures/request_error.json
98
+ - spec/filters/fixtures/request_size0_agg.json
99
+ - spec/filters/fixtures/request_x_1.json
100
+ - spec/filters/fixtures/request_x_10.json
101
+ - spec/filters/integration/opensearch_spec.rb
102
+ - spec/filters/opensearch_spec.rb
103
+ - spec/opensearch_helper.rb
104
+ homepage: https://github.com/Anarhyst266/logstash-filter-opensearch-manticore
105
+ licenses:
106
+ - Apache License (2.0)
107
+ metadata:
108
+ logstash_plugin: 'true'
109
+ logstash_group: filter
110
+ post_install_message:
111
+ rdoc_options: []
112
+ require_paths:
113
+ - lib
114
+ required_ruby_version: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: '0'
119
+ required_rubygems_version: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - ">="
122
+ - !ruby/object:Gem::Version
123
+ version: '0'
124
+ requirements: []
125
+ rubygems_version: 3.0.3.1
126
+ signing_key:
127
+ specification_version: 4
128
+ summary: Copies fields from previous log events in OpenSearch to current events
129
+ test_files:
130
+ - spec/filters/fixtures/opensearch_7.x_hits_total_as_object.json
131
+ - spec/filters/fixtures/query_template.json
132
+ - spec/filters/fixtures/query_template_unicode.json
133
+ - spec/filters/fixtures/request_error.json
134
+ - spec/filters/fixtures/request_size0_agg.json
135
+ - spec/filters/fixtures/request_x_1.json
136
+ - spec/filters/fixtures/request_x_10.json
137
+ - spec/filters/integration/opensearch_spec.rb
138
+ - spec/filters/opensearch_spec.rb
139
+ - spec/opensearch_helper.rb