logstash-input-elasticsearch 4.19.1 → 4.20.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +15 -0
- data/lib/logstash/inputs/elasticsearch/aggregation.rb +45 -0
- data/lib/logstash/inputs/elasticsearch.rb +23 -11
- data/logstash-input-elasticsearch.gemspec +1 -1
- data/spec/inputs/elasticsearch_spec.rb +66 -11
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15dab89f58c563ac371fc048ddd628929b0357a9374d30c71c3f4b7867d27412
|
4
|
+
data.tar.gz: ddc36fbfd58ee1ae909f2383033195725240041480e2c48445076e70c37360ad
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 49747a1d1a3714c9aa762a818b177806b26f1cb2935b14d912432b8ca77b8852e06a9b12421af53e9da9bf1b7ca5ee1aee58445d49aa9875729d5ba6d0e7218c
|
7
|
+
data.tar.gz: 1f22b7740c7a2dac24ed0b6e7be3ad312dd457b7f11b37121ab1260c372e492af5f53ff0d6e481c06ddb39ce6b20bf9e38e008345aaa4faff47eddbfa415f110
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 4.20.0
|
2
|
+
- Added `response_type` configuration option to allow processing result of aggregations [#202](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/202)
|
3
|
+
|
1
4
|
## 4.19.1
|
2
5
|
- Plugin version bump to pick up docs fix in [#199](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/199) required to clear build error in docgen. [#200](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/200)
|
3
6
|
|
data/docs/index.asciidoc
CHANGED
@@ -115,6 +115,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
115
115
|
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
116
116
|
| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
|
117
117
|
| <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
|
118
|
+
| <<plugins-{type}s-{plugin}-response_type>> |<<string,string>>, one of `["hits","aggregations"]`|No
|
118
119
|
| <<plugins-{type}s-{plugin}-request_timeout_seconds>> | <<number,number>>|No
|
119
120
|
| <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
|
120
121
|
| <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
|
@@ -337,6 +338,20 @@ documentation] for more information.
|
|
337
338
|
When <<plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`,
|
338
339
|
the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more.
|
339
340
|
|
341
|
+
[id="plugins-{type}s-{plugin}-response_type"]
|
342
|
+
===== `response_type`
|
343
|
+
|
344
|
+
* Value can be any of: `hits`, `aggregations`
|
345
|
+
* Default value is `hits`
|
346
|
+
|
347
|
+
Which part of the result to transform into Logstash events when processing the
|
348
|
+
response from the query.
|
349
|
+
The default `hits` will generate one event per returned document (i.e. "hit").
|
350
|
+
When set to `aggregations`, a single Logstash event will be generated with the
|
351
|
+
contents of the `aggregations` object of the query's response. In this case the
|
352
|
+
`hits` object will be ignored. The parameter `size` will be always be set to
|
353
|
+
0 regardless of the default or user-defined value set in this plugin.
|
354
|
+
|
340
355
|
[id="plugins-{type}s-{plugin}-request_timeout_seconds"]
|
341
356
|
===== `request_timeout_seconds`
|
342
357
|
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'logstash/helpers/loggable_try'
|
2
|
+
|
3
|
+
module LogStash
|
4
|
+
module Inputs
|
5
|
+
class Elasticsearch
|
6
|
+
class Aggregation
|
7
|
+
include LogStash::Util::Loggable
|
8
|
+
|
9
|
+
AGGREGATION_JOB = "aggregation"
|
10
|
+
|
11
|
+
def initialize(client, plugin)
|
12
|
+
@client = client
|
13
|
+
@plugin_params = plugin.params
|
14
|
+
|
15
|
+
@size = @plugin_params["size"]
|
16
|
+
@query = @plugin_params["query"]
|
17
|
+
@retries = @plugin_params["retries"]
|
18
|
+
@agg_options = {
|
19
|
+
:index => @index,
|
20
|
+
:size => 0
|
21
|
+
}.merge(:body => @query)
|
22
|
+
|
23
|
+
@plugin = plugin
|
24
|
+
end
|
25
|
+
|
26
|
+
def retryable(job_name, &block)
|
27
|
+
stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
|
28
|
+
stud_try.try((@retries + 1).times) { yield }
|
29
|
+
rescue => e
|
30
|
+
error_details = {:message => e.message, :cause => e.cause}
|
31
|
+
error_details[:backtrace] = e.backtrace if logger.debug?
|
32
|
+
logger.error("Tried #{job_name} unsuccessfully", error_details)
|
33
|
+
end
|
34
|
+
|
35
|
+
def do_run(output_queue)
|
36
|
+
logger.info("Aggregation starting")
|
37
|
+
r = retryable(AGGREGATION_JOB) do
|
38
|
+
@client.search(@agg_options)
|
39
|
+
end
|
40
|
+
@plugin.push_hit(r, output_queue, 'aggregations')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -74,6 +74,7 @@ require_relative "elasticsearch/patches/_elasticsearch_transport_connections_sel
|
|
74
74
|
class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
75
75
|
|
76
76
|
require 'logstash/inputs/elasticsearch/paginated_search'
|
77
|
+
require 'logstash/inputs/elasticsearch/aggregation'
|
77
78
|
|
78
79
|
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
79
80
|
include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
|
@@ -101,6 +102,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
101
102
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
102
103
|
config :query, :validate => :string, :default => '{ "sort": [ "_doc" ] }'
|
103
104
|
|
105
|
+
# This allows you to speccify the response type: either hits or aggregations
|
106
|
+
# where hits: normal search request
|
107
|
+
# aggregations: aggregation request
|
108
|
+
config :response_type, :validate => ['hits', 'aggregations'], :default => 'hits'
|
109
|
+
|
104
110
|
# This allows you to set the maximum number of hits returned per scroll.
|
105
111
|
config :size, :validate => :number, :default => 1000
|
106
112
|
|
@@ -282,11 +288,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
282
288
|
fill_hosts_from_cloud_id
|
283
289
|
setup_ssl_params!
|
284
290
|
|
285
|
-
@options = {
|
286
|
-
:index => @index,
|
287
|
-
:scroll => @scroll,
|
288
|
-
:size => @size
|
289
|
-
}
|
290
291
|
@base_query = LogStash::Json.load(@query)
|
291
292
|
if @slices
|
292
293
|
@base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
|
@@ -328,21 +329,25 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
328
329
|
|
329
330
|
setup_search_api
|
330
331
|
|
332
|
+
setup_query_executor
|
333
|
+
|
331
334
|
@client
|
332
335
|
end
|
333
336
|
|
334
|
-
|
335
337
|
def run(output_queue)
|
336
338
|
if @schedule
|
337
|
-
scheduler.cron(@schedule) { @
|
339
|
+
scheduler.cron(@schedule) { @query_executor.do_run(output_queue) }
|
338
340
|
scheduler.join
|
339
341
|
else
|
340
|
-
@
|
342
|
+
@query_executor.do_run(output_queue)
|
341
343
|
end
|
342
344
|
end
|
343
345
|
|
344
|
-
|
345
|
-
|
346
|
+
##
|
347
|
+
# This can be called externally from the query_executor
|
348
|
+
public
|
349
|
+
def push_hit(hit, output_queue, root_field = '_source')
|
350
|
+
event = targeted_event_factory.new_event hit[root_field]
|
346
351
|
set_docinfo_fields(hit, event) if @docinfo
|
347
352
|
decorate(event)
|
348
353
|
output_queue << event
|
@@ -643,13 +648,20 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
643
648
|
@search_api
|
644
649
|
end
|
645
650
|
|
651
|
+
end
|
646
652
|
|
647
|
-
|
653
|
+
def setup_query_executor
|
654
|
+
@query_executor = case @response_type
|
655
|
+
when 'hits'
|
656
|
+
if @resolved_search_api == "search_after"
|
648
657
|
LogStash::Inputs::Elasticsearch::SearchAfter.new(@client, self)
|
649
658
|
else
|
650
659
|
logger.warn("scroll API is no longer recommended for pagination. Consider using search_after instead.") if es_major_version >= 8
|
651
660
|
LogStash::Inputs::Elasticsearch::Scroll.new(@client, self)
|
652
661
|
end
|
662
|
+
when 'aggregations'
|
663
|
+
LogStash::Inputs::Elasticsearch::Aggregation.new(@client, self)
|
664
|
+
end
|
653
665
|
end
|
654
666
|
|
655
667
|
module URIOrEmptyValidator
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.
|
4
|
+
s.version = '4.20.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -112,14 +112,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
112
112
|
context "ES 8" do
|
113
113
|
let(:es_version) { "8.10.0" }
|
114
114
|
it "resolves `auto` to `search_after`" do
|
115
|
-
expect(plugin.instance_variable_get(:@
|
115
|
+
expect(plugin.instance_variable_get(:@query_executor)).to be_a LogStash::Inputs::Elasticsearch::SearchAfter
|
116
116
|
end
|
117
117
|
end
|
118
118
|
|
119
119
|
context "ES 7" do
|
120
120
|
let(:es_version) { "7.17.0" }
|
121
121
|
it "resolves `auto` to `scroll`" do
|
122
|
-
expect(plugin.instance_variable_get(:@
|
122
|
+
expect(plugin.instance_variable_get(:@query_executor)).to be_a LogStash::Inputs::Elasticsearch::Scroll
|
123
123
|
end
|
124
124
|
end
|
125
125
|
end
|
@@ -268,7 +268,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
268
268
|
before { plugin.register }
|
269
269
|
|
270
270
|
it 'runs just one slice' do
|
271
|
-
expect(plugin.instance_variable_get(:@
|
271
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).with(duck_type(:<<), nil)
|
272
272
|
expect(Thread).to_not receive(:new)
|
273
273
|
|
274
274
|
plugin.run([])
|
@@ -280,7 +280,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
280
280
|
before { plugin.register }
|
281
281
|
|
282
282
|
it 'runs just one slice' do
|
283
|
-
expect(plugin.instance_variable_get(:@
|
283
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).with(duck_type(:<<), nil)
|
284
284
|
expect(Thread).to_not receive(:new)
|
285
285
|
|
286
286
|
plugin.run([])
|
@@ -295,7 +295,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
295
295
|
it "runs #{slice_count} independent slices" do
|
296
296
|
expect(Thread).to receive(:new).and_call_original.exactly(slice_count).times
|
297
297
|
slice_count.times do |slice_id|
|
298
|
-
expect(plugin.instance_variable_get(:@
|
298
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).with(duck_type(:<<), slice_id)
|
299
299
|
end
|
300
300
|
|
301
301
|
plugin.run([])
|
@@ -423,8 +423,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
423
423
|
expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
|
424
424
|
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
|
425
425
|
|
426
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
427
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
426
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :next_page)
|
427
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :initial_search)
|
428
428
|
end
|
429
429
|
|
430
430
|
let(:client) { Elasticsearch::Client.new }
|
@@ -493,14 +493,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
493
493
|
expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
|
494
494
|
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_raise("boom")
|
495
495
|
|
496
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
497
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
496
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :next_page)
|
497
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :initial_search)
|
498
498
|
end
|
499
499
|
|
500
500
|
let(:client) { Elasticsearch::Client.new }
|
501
501
|
|
502
502
|
it 'insert event to queue without waiting other slices' do
|
503
|
-
expect(plugin.instance_variable_get(:@
|
503
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).twice.and_wrap_original do |m, *args|
|
504
504
|
q = args[0]
|
505
505
|
slice_id = args[1]
|
506
506
|
if slice_id == 0
|
@@ -1020,7 +1020,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
1020
1020
|
|
1021
1021
|
it "should properly schedule" do
|
1022
1022
|
begin
|
1023
|
-
expect(plugin.instance_variable_get(:@
|
1023
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:do_run) {
|
1024
1024
|
queue << LogStash::Event.new({})
|
1025
1025
|
}.at_least(:twice)
|
1026
1026
|
runner = Thread.start { plugin.run(queue) }
|
@@ -1033,7 +1033,62 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
1033
1033
|
runner.join if runner
|
1034
1034
|
end
|
1035
1035
|
end
|
1036
|
+
end
|
1037
|
+
|
1038
|
+
context "aggregations" do
|
1039
|
+
let(:config) do
|
1040
|
+
{
|
1041
|
+
'hosts' => ["localhost"],
|
1042
|
+
'query' => '{ "query": {}, "size": 0, "aggs":{"total_count": { "value_count": { "field": "type" }}, "empty_count": { "sum": { "field": "_meta.empty_event" }}}}',
|
1043
|
+
'response_type' => 'aggregations',
|
1044
|
+
'size' => 0
|
1045
|
+
}
|
1046
|
+
end
|
1036
1047
|
|
1048
|
+
let(:mock_response) do
|
1049
|
+
{
|
1050
|
+
"took" => 27,
|
1051
|
+
"timed_out" => false,
|
1052
|
+
"_shards" => {
|
1053
|
+
"total" => 169,
|
1054
|
+
"successful" => 169,
|
1055
|
+
"skipped" => 0,
|
1056
|
+
"failed" => 0
|
1057
|
+
},
|
1058
|
+
"hits" => {
|
1059
|
+
"total" => 10,
|
1060
|
+
"max_score" => 1.0,
|
1061
|
+
"hits" => []
|
1062
|
+
},
|
1063
|
+
"aggregations" => {
|
1064
|
+
"total_counter" => {
|
1065
|
+
"value" => 10
|
1066
|
+
},
|
1067
|
+
"empty_counter" => {
|
1068
|
+
"value" => 5
|
1069
|
+
},
|
1070
|
+
}
|
1071
|
+
}
|
1072
|
+
end
|
1073
|
+
|
1074
|
+
before(:each) do
|
1075
|
+
client = Elasticsearch::Client.new
|
1076
|
+
|
1077
|
+
expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
|
1078
|
+
expect(client).to receive(:search).with(any_args).and_return(mock_response)
|
1079
|
+
expect(client).to receive(:ping)
|
1080
|
+
end
|
1081
|
+
|
1082
|
+
before { plugin.register }
|
1083
|
+
|
1084
|
+
it 'creates the events from the aggregations' do
|
1085
|
+
plugin.run queue
|
1086
|
+
event = queue.pop
|
1087
|
+
|
1088
|
+
expect(event).to be_a(LogStash::Event)
|
1089
|
+
expect(event.get("[total_counter][value]")).to eql 10
|
1090
|
+
expect(event.get("[empty_counter][value]")).to eql 5
|
1091
|
+
end
|
1037
1092
|
end
|
1038
1093
|
|
1039
1094
|
context "retries" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.20.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -271,6 +271,7 @@ files:
|
|
271
271
|
- docs/index.asciidoc
|
272
272
|
- lib/logstash/helpers/loggable_try.rb
|
273
273
|
- lib/logstash/inputs/elasticsearch.rb
|
274
|
+
- lib/logstash/inputs/elasticsearch/aggregation.rb
|
274
275
|
- lib/logstash/inputs/elasticsearch/paginated_search.rb
|
275
276
|
- lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
|
276
277
|
- lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
|