logstash-input-elasticsearch 4.1.1 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/LICENSE +1 -1
- data/docs/index.asciidoc +42 -2
- data/lib/logstash/inputs/elasticsearch.rb +48 -2
- data/logstash-input-elasticsearch.gemspec +7 -1
- data/spec/inputs/elasticsearch_spec.rb +74 -1
- metadata +87 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c800b8e0032e2d43897b1c4e0dab3cf4d6d22c14919fcdefe7b7df8ec4a64ffa
|
4
|
+
data.tar.gz: 063e672f41db0c0711c7d2229fd05b5dbef0d28fed9f34afe5c2c2ad57ff1dc2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 63503ac4073666f88eca84c6af52d2598ba425b4ca7b16ec6c77bcb97def7083119943ea7f940536bd49a8cad581d2afb502634a00e336a223f2abff9842038d
|
7
|
+
data.tar.gz: 5d2ec1e87c76a79f590d3f93dc8e2667597b74aba4f1503282f75de34ce98e6c6cac4d49ddcc86668669eb7706dc97a312fb93f7bad3b88fa2ff98608cd9bc94
|
data/CHANGELOG.md
CHANGED
data/LICENSE
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -32,6 +32,9 @@ plugin to version 4.0.2 or higher.
|
|
32
32
|
|
33
33
|
Read from an Elasticsearch cluster, based on search query results.
|
34
34
|
This is useful for replaying test logs, reindexing, etc.
|
35
|
+
You can periodically schedule ingestion using a cron syntax
|
36
|
+
(see `schedule` setting) or run the query one time to load
|
37
|
+
data into Logstash.
|
35
38
|
|
36
39
|
Example:
|
37
40
|
[source,ruby]
|
@@ -55,6 +58,25 @@ This would create an Elasticsearch query with the following format:
|
|
55
58
|
}'
|
56
59
|
|
57
60
|
|
61
|
+
==== Scheduling
|
62
|
+
|
63
|
+
Input from this plugin can be scheduled to run periodically according to a specific
|
64
|
+
schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler].
|
65
|
+
The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ).
|
66
|
+
|
67
|
+
Examples:
|
68
|
+
|
69
|
+
|==========================================================
|
70
|
+
| `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March.
|
71
|
+
| `0 * * * *` | will execute on the 0th minute of every hour every day.
|
72
|
+
| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day.
|
73
|
+
|==========================================================
|
74
|
+
|
75
|
+
|
76
|
+
Further documentation describing this syntax can be found
|
77
|
+
https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
|
78
|
+
|
79
|
+
|
58
80
|
[id="plugins-{type}s-{plugin}-options"]
|
59
81
|
==== Elasticsearch Input Configuration Options
|
60
82
|
|
@@ -71,6 +93,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
71
93
|
| <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
|
72
94
|
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
73
95
|
| <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
|
96
|
+
| <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
|
74
97
|
| <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
|
75
98
|
| <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
|
76
99
|
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
|
@@ -127,6 +150,11 @@ Example
|
|
127
150
|
}
|
128
151
|
|
129
152
|
|
153
|
+
NOTE: Starting with Logstash 6.0, the `document_type` option is
|
154
|
+
deprecated due to the
|
155
|
+
https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Logstash 6.0].
|
156
|
+
It will be removed in the next major version of Logstash.
|
157
|
+
|
130
158
|
[id="plugins-{type}s-{plugin}-docinfo_fields"]
|
131
159
|
===== `docinfo_fields`
|
132
160
|
|
@@ -136,7 +164,7 @@ Example
|
|
136
164
|
If document metadata storage is requested by enabling the `docinfo`
|
137
165
|
option, this option lists the metadata fields to save in the current
|
138
166
|
event. See
|
139
|
-
|
167
|
+
http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html[Document Metadata]
|
140
168
|
in the Elasticsearch documentation for more information.
|
141
169
|
|
142
170
|
[id="plugins-{type}s-{plugin}-docinfo_target"]
|
@@ -184,9 +212,21 @@ string authentication will be disabled.
|
|
184
212
|
* Default value is `'{ "sort": [ "_doc" ] }'`
|
185
213
|
|
186
214
|
The query to be executed. Read the
|
187
|
-
|
215
|
+
https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Elasticsearch query DSL documentation]
|
188
216
|
for more information.
|
189
217
|
|
218
|
+
[id="plugins-{type}s-{plugin}-schedule"]
|
219
|
+
===== `schedule`
|
220
|
+
|
221
|
+
* Value type is <<string,string>>
|
222
|
+
* There is no default value for this setting.
|
223
|
+
|
224
|
+
Schedule of when to periodically run statement, in Cron format
|
225
|
+
for example: "* * * * *" (execute query every minute, on the minute)
|
226
|
+
|
227
|
+
There is no schedule by default. If no schedule is given, then the statement is run
|
228
|
+
exactly once.
|
229
|
+
|
190
230
|
[id="plugins-{type}s-{plugin}-scroll"]
|
191
231
|
===== `scroll`
|
192
232
|
|
@@ -15,6 +15,8 @@ require "base64"
|
|
15
15
|
#
|
16
16
|
# Read from an Elasticsearch cluster, based on search query results.
|
17
17
|
# This is useful for replaying test logs, reindexing, etc.
|
18
|
+
# It also supports periodically scheduling lookup enrichments
|
19
|
+
# using a cron syntax (see `schedule` setting).
|
18
20
|
#
|
19
21
|
# Example:
|
20
22
|
# [source,ruby]
|
@@ -37,6 +39,24 @@ require "base64"
|
|
37
39
|
# "sort": [ "_doc" ]
|
38
40
|
# }'
|
39
41
|
#
|
42
|
+
# ==== Scheduling
|
43
|
+
#
|
44
|
+
# Input from this plugin can be scheduled to run periodically according to a specific
|
45
|
+
# schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler].
|
46
|
+
# The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ).
|
47
|
+
#
|
48
|
+
# Examples:
|
49
|
+
#
|
50
|
+
# |==========================================================
|
51
|
+
# | `* 5 * 1-3 *` | will execute every minute of 5am every day of January through March.
|
52
|
+
# | `0 * * * *` | will execute on the 0th minute of every hour every day.
|
53
|
+
# | `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day.
|
54
|
+
# |==========================================================
|
55
|
+
#
|
56
|
+
#
|
57
|
+
# Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
|
58
|
+
#
|
59
|
+
#
|
40
60
|
class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
41
61
|
config_name "elasticsearch"
|
42
62
|
|
@@ -114,8 +134,16 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
114
134
|
# SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary
|
115
135
|
config :ca_file, :validate => :path
|
116
136
|
|
137
|
+
# Schedule of when to periodically run statement, in Cron format
|
138
|
+
# for example: "* * * * *" (execute query every minute, on the minute)
|
139
|
+
#
|
140
|
+
# There is no schedule by default. If no schedule is given, then the statement is run
|
141
|
+
# exactly once.
|
142
|
+
config :schedule, :validate => :string
|
143
|
+
|
117
144
|
def register
|
118
145
|
require "elasticsearch"
|
146
|
+
require "rufus/scheduler"
|
119
147
|
|
120
148
|
@options = {
|
121
149
|
:index => @index,
|
@@ -147,7 +175,27 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
147
175
|
@client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
|
148
176
|
end
|
149
177
|
|
178
|
+
|
150
179
|
def run(output_queue)
|
180
|
+
if @schedule
|
181
|
+
@scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
|
182
|
+
@scheduler.cron @schedule do
|
183
|
+
do_run(output_queue)
|
184
|
+
end
|
185
|
+
|
186
|
+
@scheduler.join
|
187
|
+
else
|
188
|
+
do_run(output_queue)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
def stop
|
193
|
+
@scheduler.stop if @scheduler
|
194
|
+
end
|
195
|
+
|
196
|
+
private
|
197
|
+
|
198
|
+
def do_run(output_queue)
|
151
199
|
# get first wave of data
|
152
200
|
r = @client.search(@options)
|
153
201
|
|
@@ -160,8 +208,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
160
208
|
end
|
161
209
|
end
|
162
210
|
|
163
|
-
private
|
164
|
-
|
165
211
|
def process_next_scroll(output_queue, scroll_id)
|
166
212
|
r = scroll_request(scroll_id)
|
167
213
|
r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.
|
4
|
+
s.version = '4.2.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -25,6 +25,12 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.add_runtime_dependency 'elasticsearch', ['>= 5.0.3', '< 6.0.0']
|
26
26
|
|
27
27
|
s.add_runtime_dependency 'logstash-codec-json'
|
28
|
+
s.add_runtime_dependency 'logstash-codec-plain'
|
29
|
+
s.add_runtime_dependency 'sequel'
|
30
|
+
s.add_runtime_dependency 'tzinfo'
|
31
|
+
s.add_runtime_dependency 'tzinfo-data'
|
32
|
+
s.add_runtime_dependency 'rufus-scheduler'
|
28
33
|
|
29
34
|
s.add_development_dependency 'logstash-devutils'
|
35
|
+
s.add_development_dependency 'timecop'
|
30
36
|
end
|
@@ -2,12 +2,23 @@
|
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/inputs/elasticsearch"
|
4
4
|
require "elasticsearch"
|
5
|
+
require "timecop"
|
6
|
+
require "stud/temporary"
|
7
|
+
require "time"
|
8
|
+
require "date"
|
5
9
|
|
6
10
|
describe LogStash::Inputs::Elasticsearch do
|
7
11
|
|
12
|
+
let(:plugin) { LogStash::Inputs::Elasticsearch.new(config) }
|
13
|
+
let(:queue) { Queue.new }
|
14
|
+
|
8
15
|
it_behaves_like "an interruptible input plugin" do
|
9
16
|
let(:esclient) { double("elasticsearch-client") }
|
10
|
-
let(:config)
|
17
|
+
let(:config) do
|
18
|
+
{
|
19
|
+
"schedule" => "* * * * * UTC"
|
20
|
+
}
|
21
|
+
end
|
11
22
|
|
12
23
|
before :each do
|
13
24
|
allow(Elasticsearch::Client).to receive(:new).and_return(esclient)
|
@@ -271,4 +282,66 @@ describe LogStash::Inputs::Elasticsearch do
|
|
271
282
|
end
|
272
283
|
end
|
273
284
|
end
|
285
|
+
|
286
|
+
context "when scheduling" do
|
287
|
+
let(:config) do
|
288
|
+
{
|
289
|
+
"hosts" => ["localhost"],
|
290
|
+
"query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
|
291
|
+
"schedule" => "* * * * * UTC"
|
292
|
+
}
|
293
|
+
end
|
294
|
+
|
295
|
+
response = {
|
296
|
+
"_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
|
297
|
+
"took" => 27,
|
298
|
+
"timed_out" => false,
|
299
|
+
"_shards" => {
|
300
|
+
"total" => 169,
|
301
|
+
"successful" => 169,
|
302
|
+
"failed" => 0
|
303
|
+
},
|
304
|
+
"hits" => {
|
305
|
+
"total" => 1,
|
306
|
+
"max_score" => 1.0,
|
307
|
+
"hits" => [ {
|
308
|
+
"_index" => "logstash-2014.10.12",
|
309
|
+
"_type" => "logs",
|
310
|
+
"_id" => "C5b2xLQwTZa76jBmHIbwHQ",
|
311
|
+
"_score" => 1.0,
|
312
|
+
"_source" => { "message" => ["ohayo"] }
|
313
|
+
} ]
|
314
|
+
}
|
315
|
+
}
|
316
|
+
|
317
|
+
scroll_reponse = {
|
318
|
+
"_scroll_id" => "r453Wc1jh0caLJhSDg",
|
319
|
+
"hits" => { "hits" => [] }
|
320
|
+
}
|
321
|
+
|
322
|
+
before do
|
323
|
+
plugin.register
|
324
|
+
end
|
325
|
+
|
326
|
+
it "should properly schedule" do
|
327
|
+
|
328
|
+
Timecop.travel(Time.new(2000))
|
329
|
+
Timecop.scale(60)
|
330
|
+
runner = Thread.new do
|
331
|
+
expect(plugin).to receive(:do_run) {
|
332
|
+
queue << LogStash::Event.new({})
|
333
|
+
}.at_least(:twice)
|
334
|
+
|
335
|
+
plugin.run(queue)
|
336
|
+
end
|
337
|
+
sleep 3
|
338
|
+
plugin.stop
|
339
|
+
runner.kill
|
340
|
+
runner.join
|
341
|
+
expect(queue.size).to eq(2)
|
342
|
+
Timecop.return
|
343
|
+
end
|
344
|
+
|
345
|
+
end
|
346
|
+
|
274
347
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -64,6 +64,76 @@ dependencies:
|
|
64
64
|
- - ">="
|
65
65
|
- !ruby/object:Gem::Version
|
66
66
|
version: '0'
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
requirement: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '0'
|
73
|
+
name: logstash-codec-plain
|
74
|
+
prerelease: false
|
75
|
+
type: :runtime
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
requirement: !ruby/object:Gem::Requirement
|
83
|
+
requirements:
|
84
|
+
- - ">="
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
version: '0'
|
87
|
+
name: sequel
|
88
|
+
prerelease: false
|
89
|
+
type: :runtime
|
90
|
+
version_requirements: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '0'
|
101
|
+
name: tzinfo
|
102
|
+
prerelease: false
|
103
|
+
type: :runtime
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
- !ruby/object:Gem::Dependency
|
110
|
+
requirement: !ruby/object:Gem::Requirement
|
111
|
+
requirements:
|
112
|
+
- - ">="
|
113
|
+
- !ruby/object:Gem::Version
|
114
|
+
version: '0'
|
115
|
+
name: tzinfo-data
|
116
|
+
prerelease: false
|
117
|
+
type: :runtime
|
118
|
+
version_requirements: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
- !ruby/object:Gem::Dependency
|
124
|
+
requirement: !ruby/object:Gem::Requirement
|
125
|
+
requirements:
|
126
|
+
- - ">="
|
127
|
+
- !ruby/object:Gem::Version
|
128
|
+
version: '0'
|
129
|
+
name: rufus-scheduler
|
130
|
+
prerelease: false
|
131
|
+
type: :runtime
|
132
|
+
version_requirements: !ruby/object:Gem::Requirement
|
133
|
+
requirements:
|
134
|
+
- - ">="
|
135
|
+
- !ruby/object:Gem::Version
|
136
|
+
version: '0'
|
67
137
|
- !ruby/object:Gem::Dependency
|
68
138
|
requirement: !ruby/object:Gem::Requirement
|
69
139
|
requirements:
|
@@ -78,6 +148,20 @@ dependencies:
|
|
78
148
|
- - ">="
|
79
149
|
- !ruby/object:Gem::Version
|
80
150
|
version: '0'
|
151
|
+
- !ruby/object:Gem::Dependency
|
152
|
+
requirement: !ruby/object:Gem::Requirement
|
153
|
+
requirements:
|
154
|
+
- - ">="
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: '0'
|
157
|
+
name: timecop
|
158
|
+
prerelease: false
|
159
|
+
type: :development
|
160
|
+
version_requirements: !ruby/object:Gem::Requirement
|
161
|
+
requirements:
|
162
|
+
- - ">="
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '0'
|
81
165
|
description: This gem is a Logstash plugin required to be installed on top of the
|
82
166
|
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
83
167
|
gem is not a stand-alone program
|
@@ -118,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
118
202
|
version: '0'
|
119
203
|
requirements: []
|
120
204
|
rubyforge_project:
|
121
|
-
rubygems_version: 2.6.
|
205
|
+
rubygems_version: 2.6.13
|
122
206
|
signing_key:
|
123
207
|
specification_version: 4
|
124
208
|
summary: Reads query results from an Elasticsearch cluster
|