logstash-input-elasticsearch 4.19.1 → 4.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +15 -0
- data/lib/logstash/inputs/elasticsearch/aggregation.rb +45 -0
- data/lib/logstash/inputs/elasticsearch.rb +23 -11
- data/logstash-input-elasticsearch.gemspec +1 -1
- data/spec/inputs/elasticsearch_spec.rb +66 -11
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 15dab89f58c563ac371fc048ddd628929b0357a9374d30c71c3f4b7867d27412
|
4
|
+
data.tar.gz: ddc36fbfd58ee1ae909f2383033195725240041480e2c48445076e70c37360ad
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 49747a1d1a3714c9aa762a818b177806b26f1cb2935b14d912432b8ca77b8852e06a9b12421af53e9da9bf1b7ca5ee1aee58445d49aa9875729d5ba6d0e7218c
|
7
|
+
data.tar.gz: 1f22b7740c7a2dac24ed0b6e7be3ad312dd457b7f11b37121ab1260c372e492af5f53ff0d6e481c06ddb39ce6b20bf9e38e008345aaa4faff47eddbfa415f110
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 4.20.0
|
2
|
+
- Added `response_type` configuration option to allow processing result of aggregations [#202](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/202)
|
3
|
+
|
1
4
|
## 4.19.1
|
2
5
|
- Plugin version bump to pick up docs fix in [#199](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/199) required to clear build error in docgen. [#200](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/200)
|
3
6
|
|
data/docs/index.asciidoc
CHANGED
@@ -115,6 +115,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
115
115
|
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
116
116
|
| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
|
117
117
|
| <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
|
118
|
+
| <<plugins-{type}s-{plugin}-response_type>> |<<string,string>>, one of `["hits","aggregations"]`|No
|
118
119
|
| <<plugins-{type}s-{plugin}-request_timeout_seconds>> | <<number,number>>|No
|
119
120
|
| <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
|
120
121
|
| <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
|
@@ -337,6 +338,20 @@ documentation] for more information.
|
|
337
338
|
When <<plugins-{type}s-{plugin}-search_api>> resolves to `search_after` and the query does not specify `sort`,
|
338
339
|
the default sort `'{ "sort": { "_shard_doc": "asc" } }'` will be added to the query. Please refer to the {ref}/paginate-search-results.html#search-after[Elasticsearch search_after] parameter to know more.
|
339
340
|
|
341
|
+
[id="plugins-{type}s-{plugin}-response_type"]
|
342
|
+
===== `response_type`
|
343
|
+
|
344
|
+
* Value can be any of: `hits`, `aggregations`
|
345
|
+
* Default value is `hits`
|
346
|
+
|
347
|
+
Which part of the result to transform into Logstash events when processing the
|
348
|
+
response from the query.
|
349
|
+
The default `hits` will generate one event per returned document (i.e. "hit").
|
350
|
+
When set to `aggregations`, a single Logstash event will be generated with the
|
351
|
+
contents of the `aggregations` object of the query's response. In this case the
|
352
|
+
`hits` object will be ignored. The parameter `size` will be always be set to
|
353
|
+
0 regardless of the default or user-defined value set in this plugin.
|
354
|
+
|
340
355
|
[id="plugins-{type}s-{plugin}-request_timeout_seconds"]
|
341
356
|
===== `request_timeout_seconds`
|
342
357
|
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'logstash/helpers/loggable_try'
|
2
|
+
|
3
|
+
module LogStash
|
4
|
+
module Inputs
|
5
|
+
class Elasticsearch
|
6
|
+
class Aggregation
|
7
|
+
include LogStash::Util::Loggable
|
8
|
+
|
9
|
+
AGGREGATION_JOB = "aggregation"
|
10
|
+
|
11
|
+
def initialize(client, plugin)
|
12
|
+
@client = client
|
13
|
+
@plugin_params = plugin.params
|
14
|
+
|
15
|
+
@size = @plugin_params["size"]
|
16
|
+
@query = @plugin_params["query"]
|
17
|
+
@retries = @plugin_params["retries"]
|
18
|
+
@agg_options = {
|
19
|
+
:index => @index,
|
20
|
+
:size => 0
|
21
|
+
}.merge(:body => @query)
|
22
|
+
|
23
|
+
@plugin = plugin
|
24
|
+
end
|
25
|
+
|
26
|
+
def retryable(job_name, &block)
|
27
|
+
stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
|
28
|
+
stud_try.try((@retries + 1).times) { yield }
|
29
|
+
rescue => e
|
30
|
+
error_details = {:message => e.message, :cause => e.cause}
|
31
|
+
error_details[:backtrace] = e.backtrace if logger.debug?
|
32
|
+
logger.error("Tried #{job_name} unsuccessfully", error_details)
|
33
|
+
end
|
34
|
+
|
35
|
+
def do_run(output_queue)
|
36
|
+
logger.info("Aggregation starting")
|
37
|
+
r = retryable(AGGREGATION_JOB) do
|
38
|
+
@client.search(@agg_options)
|
39
|
+
end
|
40
|
+
@plugin.push_hit(r, output_queue, 'aggregations')
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -74,6 +74,7 @@ require_relative "elasticsearch/patches/_elasticsearch_transport_connections_sel
|
|
74
74
|
class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
75
75
|
|
76
76
|
require 'logstash/inputs/elasticsearch/paginated_search'
|
77
|
+
require 'logstash/inputs/elasticsearch/aggregation'
|
77
78
|
|
78
79
|
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
79
80
|
include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
|
@@ -101,6 +102,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
101
102
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
102
103
|
config :query, :validate => :string, :default => '{ "sort": [ "_doc" ] }'
|
103
104
|
|
105
|
+
# This allows you to speccify the response type: either hits or aggregations
|
106
|
+
# where hits: normal search request
|
107
|
+
# aggregations: aggregation request
|
108
|
+
config :response_type, :validate => ['hits', 'aggregations'], :default => 'hits'
|
109
|
+
|
104
110
|
# This allows you to set the maximum number of hits returned per scroll.
|
105
111
|
config :size, :validate => :number, :default => 1000
|
106
112
|
|
@@ -282,11 +288,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
282
288
|
fill_hosts_from_cloud_id
|
283
289
|
setup_ssl_params!
|
284
290
|
|
285
|
-
@options = {
|
286
|
-
:index => @index,
|
287
|
-
:scroll => @scroll,
|
288
|
-
:size => @size
|
289
|
-
}
|
290
291
|
@base_query = LogStash::Json.load(@query)
|
291
292
|
if @slices
|
292
293
|
@base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
|
@@ -328,21 +329,25 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
328
329
|
|
329
330
|
setup_search_api
|
330
331
|
|
332
|
+
setup_query_executor
|
333
|
+
|
331
334
|
@client
|
332
335
|
end
|
333
336
|
|
334
|
-
|
335
337
|
def run(output_queue)
|
336
338
|
if @schedule
|
337
|
-
scheduler.cron(@schedule) { @
|
339
|
+
scheduler.cron(@schedule) { @query_executor.do_run(output_queue) }
|
338
340
|
scheduler.join
|
339
341
|
else
|
340
|
-
@
|
342
|
+
@query_executor.do_run(output_queue)
|
341
343
|
end
|
342
344
|
end
|
343
345
|
|
344
|
-
|
345
|
-
|
346
|
+
##
|
347
|
+
# This can be called externally from the query_executor
|
348
|
+
public
|
349
|
+
def push_hit(hit, output_queue, root_field = '_source')
|
350
|
+
event = targeted_event_factory.new_event hit[root_field]
|
346
351
|
set_docinfo_fields(hit, event) if @docinfo
|
347
352
|
decorate(event)
|
348
353
|
output_queue << event
|
@@ -643,13 +648,20 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
643
648
|
@search_api
|
644
649
|
end
|
645
650
|
|
651
|
+
end
|
646
652
|
|
647
|
-
|
653
|
+
def setup_query_executor
|
654
|
+
@query_executor = case @response_type
|
655
|
+
when 'hits'
|
656
|
+
if @resolved_search_api == "search_after"
|
648
657
|
LogStash::Inputs::Elasticsearch::SearchAfter.new(@client, self)
|
649
658
|
else
|
650
659
|
logger.warn("scroll API is no longer recommended for pagination. Consider using search_after instead.") if es_major_version >= 8
|
651
660
|
LogStash::Inputs::Elasticsearch::Scroll.new(@client, self)
|
652
661
|
end
|
662
|
+
when 'aggregations'
|
663
|
+
LogStash::Inputs::Elasticsearch::Aggregation.new(@client, self)
|
664
|
+
end
|
653
665
|
end
|
654
666
|
|
655
667
|
module URIOrEmptyValidator
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.
|
4
|
+
s.version = '4.20.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -112,14 +112,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
112
112
|
context "ES 8" do
|
113
113
|
let(:es_version) { "8.10.0" }
|
114
114
|
it "resolves `auto` to `search_after`" do
|
115
|
-
expect(plugin.instance_variable_get(:@
|
115
|
+
expect(plugin.instance_variable_get(:@query_executor)).to be_a LogStash::Inputs::Elasticsearch::SearchAfter
|
116
116
|
end
|
117
117
|
end
|
118
118
|
|
119
119
|
context "ES 7" do
|
120
120
|
let(:es_version) { "7.17.0" }
|
121
121
|
it "resolves `auto` to `scroll`" do
|
122
|
-
expect(plugin.instance_variable_get(:@
|
122
|
+
expect(plugin.instance_variable_get(:@query_executor)).to be_a LogStash::Inputs::Elasticsearch::Scroll
|
123
123
|
end
|
124
124
|
end
|
125
125
|
end
|
@@ -268,7 +268,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
268
268
|
before { plugin.register }
|
269
269
|
|
270
270
|
it 'runs just one slice' do
|
271
|
-
expect(plugin.instance_variable_get(:@
|
271
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).with(duck_type(:<<), nil)
|
272
272
|
expect(Thread).to_not receive(:new)
|
273
273
|
|
274
274
|
plugin.run([])
|
@@ -280,7 +280,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
280
280
|
before { plugin.register }
|
281
281
|
|
282
282
|
it 'runs just one slice' do
|
283
|
-
expect(plugin.instance_variable_get(:@
|
283
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).with(duck_type(:<<), nil)
|
284
284
|
expect(Thread).to_not receive(:new)
|
285
285
|
|
286
286
|
plugin.run([])
|
@@ -295,7 +295,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
295
295
|
it "runs #{slice_count} independent slices" do
|
296
296
|
expect(Thread).to receive(:new).and_call_original.exactly(slice_count).times
|
297
297
|
slice_count.times do |slice_id|
|
298
|
-
expect(plugin.instance_variable_get(:@
|
298
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).with(duck_type(:<<), slice_id)
|
299
299
|
end
|
300
300
|
|
301
301
|
plugin.run([])
|
@@ -423,8 +423,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
423
423
|
expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
|
424
424
|
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
|
425
425
|
|
426
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
427
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
426
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :next_page)
|
427
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :initial_search)
|
428
428
|
end
|
429
429
|
|
430
430
|
let(:client) { Elasticsearch::Client.new }
|
@@ -493,14 +493,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
493
493
|
expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
|
494
494
|
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_raise("boom")
|
495
495
|
|
496
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
497
|
-
synchronize_method!(plugin.instance_variable_get(:@
|
496
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :next_page)
|
497
|
+
synchronize_method!(plugin.instance_variable_get(:@query_executor), :initial_search)
|
498
498
|
end
|
499
499
|
|
500
500
|
let(:client) { Elasticsearch::Client.new }
|
501
501
|
|
502
502
|
it 'insert event to queue without waiting other slices' do
|
503
|
-
expect(plugin.instance_variable_get(:@
|
503
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:search).twice.and_wrap_original do |m, *args|
|
504
504
|
q = args[0]
|
505
505
|
slice_id = args[1]
|
506
506
|
if slice_id == 0
|
@@ -1020,7 +1020,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
1020
1020
|
|
1021
1021
|
it "should properly schedule" do
|
1022
1022
|
begin
|
1023
|
-
expect(plugin.instance_variable_get(:@
|
1023
|
+
expect(plugin.instance_variable_get(:@query_executor)).to receive(:do_run) {
|
1024
1024
|
queue << LogStash::Event.new({})
|
1025
1025
|
}.at_least(:twice)
|
1026
1026
|
runner = Thread.start { plugin.run(queue) }
|
@@ -1033,7 +1033,62 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
1033
1033
|
runner.join if runner
|
1034
1034
|
end
|
1035
1035
|
end
|
1036
|
+
end
|
1037
|
+
|
1038
|
+
context "aggregations" do
|
1039
|
+
let(:config) do
|
1040
|
+
{
|
1041
|
+
'hosts' => ["localhost"],
|
1042
|
+
'query' => '{ "query": {}, "size": 0, "aggs":{"total_count": { "value_count": { "field": "type" }}, "empty_count": { "sum": { "field": "_meta.empty_event" }}}}',
|
1043
|
+
'response_type' => 'aggregations',
|
1044
|
+
'size' => 0
|
1045
|
+
}
|
1046
|
+
end
|
1036
1047
|
|
1048
|
+
let(:mock_response) do
|
1049
|
+
{
|
1050
|
+
"took" => 27,
|
1051
|
+
"timed_out" => false,
|
1052
|
+
"_shards" => {
|
1053
|
+
"total" => 169,
|
1054
|
+
"successful" => 169,
|
1055
|
+
"skipped" => 0,
|
1056
|
+
"failed" => 0
|
1057
|
+
},
|
1058
|
+
"hits" => {
|
1059
|
+
"total" => 10,
|
1060
|
+
"max_score" => 1.0,
|
1061
|
+
"hits" => []
|
1062
|
+
},
|
1063
|
+
"aggregations" => {
|
1064
|
+
"total_counter" => {
|
1065
|
+
"value" => 10
|
1066
|
+
},
|
1067
|
+
"empty_counter" => {
|
1068
|
+
"value" => 5
|
1069
|
+
},
|
1070
|
+
}
|
1071
|
+
}
|
1072
|
+
end
|
1073
|
+
|
1074
|
+
before(:each) do
|
1075
|
+
client = Elasticsearch::Client.new
|
1076
|
+
|
1077
|
+
expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
|
1078
|
+
expect(client).to receive(:search).with(any_args).and_return(mock_response)
|
1079
|
+
expect(client).to receive(:ping)
|
1080
|
+
end
|
1081
|
+
|
1082
|
+
before { plugin.register }
|
1083
|
+
|
1084
|
+
it 'creates the events from the aggregations' do
|
1085
|
+
plugin.run queue
|
1086
|
+
event = queue.pop
|
1087
|
+
|
1088
|
+
expect(event).to be_a(LogStash::Event)
|
1089
|
+
expect(event.get("[total_counter][value]")).to eql 10
|
1090
|
+
expect(event.get("[empty_counter][value]")).to eql 5
|
1091
|
+
end
|
1037
1092
|
end
|
1038
1093
|
|
1039
1094
|
context "retries" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.20.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -271,6 +271,7 @@ files:
|
|
271
271
|
- docs/index.asciidoc
|
272
272
|
- lib/logstash/helpers/loggable_try.rb
|
273
273
|
- lib/logstash/inputs/elasticsearch.rb
|
274
|
+
- lib/logstash/inputs/elasticsearch/aggregation.rb
|
274
275
|
- lib/logstash/inputs/elasticsearch/paginated_search.rb
|
275
276
|
- lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
|
276
277
|
- lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
|